From 6dc8ed2391cce156418cf6c6cce3f44cbe62e99b Mon Sep 17 00:00:00 2001 From: yuyang wang <39869597+Jackson-Wang-7@users.noreply.github.com> Date: Wed, 16 Aug 2023 10:29:12 +0800 Subject: [PATCH] Remove the usage of must_cache and async_cache ### What changes are proposed in this pull request? current version didn't support must_cache and async_cache, so remove the usage of must_cache and async_cache usage in code and mark deprecated to prevent any use. ### Why are the changes needed? This change focuses on removing the write type of MUST_CACHE and ASYNC_CACHE, and https://github.com/Alluxio/alluxio/pull/17980 would remove the master-side code about async persist. ### Does this PR introduce any user facing changes? No pr-link: Alluxio/alluxio#17963 change-id: cid-14d40a217134beb8a5e7316b050af01bca18b65e --- .../src/main/proto/grpc/block_worker.proto | 2 +- .../main/proto/grpc/file_system_master.proto | 14 +-- .../block/stream/BlockWorkerDataWriter.java | 6 +- .../client/block/stream/GrpcDataWriter.java | 2 - .../client/file/FileSystemTestUtils.java | 24 +++++ .../java/alluxio/client/UnderStorageType.java | 10 --- .../main/java/alluxio/client/WriteType.java | 24 ++--- .../alluxio/master/file/meta/InodeTree.java | 4 - .../file/FileSystemMasterFsOptsTest.java | 19 ++-- .../master/file/FileSystemMasterTest.java | 36 ++------ .../master/file/FileSystemMasterTestBase.java | 4 +- .../master/journal/sink/JournalSinkTest.java | 5 ++ .../alluxio/worker/grpc/WriteRequest.java | 11 --- .../migrate/MigrateDefinitionRunTaskTest.java | 18 ---- .../src/main/java/alluxio/cli/TestRunner.java | 4 +- ...ackwardsCompatibilityJournalGenerator.java | 2 - .../alluxio/client/cli/JournalToolTest.java | 2 +- .../cli/fs/AbstractFileSystemShellTest.java | 4 +- .../cli/fs/FileSystemShellUtilsTest.java | 2 +- .../fs/command/CatCommandIntegrationTest.java | 2 +- .../ChecksumCommandIntegrationTest.java | 9 +- .../command/ChgrpCommandIntegrationTest.java | 15 ++-- .../command/ChmodCommandIntegrationTest.java | 14 ++- .../command/ChownCommandIntegrationTest.java | 22 ++--- .../CopyFromLocalCommandIntegrationTest.java | 20 ----- .../CopyToLocalCommandIntegrationTest.java | 9 +- .../fs/command/CpCommandIntegrationTest.java | 4 +- .../command/FreeCommandIntegrationTest.java | 21 ----- .../command/HeadCommandIntegrationTest.java | 13 ++- .../fs/command/LsCommandIntegrationTest.java | 40 ++++----- .../LsCommandSecurityIntegrationTest.java | 8 +- .../OldLoadCommandIntegrationTest.java | 2 +- .../command/TailCommandIntegrationTest.java | 10 +-- .../command/TestCommandIntegrationTest.java | 22 ++--- .../client/cli/fs/command/UnsetTtlTest.java | 2 +- .../command/ReportCommandIntegrationTest.java | 5 ++ .../ReportEmbeddedJournalIntegrationTest.java | 3 + .../MetricsCommandIntegrationTest.java | 2 +- .../client/fs/FileSystemIntegrationTest.java | 4 +- .../fs/FileSystemMasterIntegrationTest.java | 8 +- .../fs/FileSystemUtilsIntegrationTest.java | 30 ------- .../fs/LoadMetadataIntegrationTest.java | 2 +- .../fs/LocalFirstPolicyIntegrationTest.java | 5 +- .../fs/PersistPermissionIntegrationTest.java | 62 ------------- .../alluxio/client/fs/PinIntegrationTest.java | 2 +- .../client/fs/RemoteReadIntegrationTest.java | 2 +- .../client/fs/UfsSyncIntegrationTest.java | 38 -------- ...ConcurrentFileInStreamIntegrationTest.java | 2 +- ...FileSystemMasterCreateIntegrationTest.java | 32 ------- .../BufferedBlockInStreamIntegrationTest.java | 2 - ...leInStreamDecommissionIntegrationTest.java | 51 ----------- .../fs/io/FileInStreamIntegrationTest.java | 5 -- ...eOutStreamDecommissionIntegrationTest.java | 90 ------------------- .../fs/io/FileOutStreamIntegrationTest.java | 2 - .../io/LocalBlockInStreamIntegrationTest.java | 4 - .../fuse/AbstractFuseIntegrationTest.java | 11 ++- ...AbstractFuseFileStreamIntegrationTest.java | 3 +- .../alluxio/client/rest/ListStatusTest.java | 2 - .../client/rest/S3ClientRestApiTest.java | 2 +- .../BlockMasterJournalIntegrationTest.java | 5 +- .../journal/JournalBackupIntegrationTest.java | 6 +- .../ft/journal/JournalIntegrationTest.java | 2 +- .../JournalShutdownIntegrationTest.java | 2 +- .../ufs/UfsJournalIntegrationTest.java | 2 +- 64 files changed, 172 insertions(+), 620 deletions(-) diff --git a/common/transport/src/main/proto/grpc/block_worker.proto b/common/transport/src/main/proto/grpc/block_worker.proto index c5295302a30a..424b52ab9f59 100644 --- a/common/transport/src/main/proto/grpc/block_worker.proto +++ b/common/transport/src/main/proto/grpc/block_worker.proto @@ -313,7 +313,7 @@ message CopyResponse { message WriteOptions { optional bool overwrite = 1; - optional grpc.file.WritePType write_type = 2; + optional grpc.file.WritePType write_type = 2 [default = CACHE_THROUGH]; optional bool check_content = 3; } diff --git a/common/transport/src/main/proto/grpc/file_system_master.proto b/common/transport/src/main/proto/grpc/file_system_master.proto index 18716861d019..f4f7cc0574d1 100644 --- a/common/transport/src/main/proto/grpc/file_system_master.proto +++ b/common/transport/src/main/proto/grpc/file_system_master.proto @@ -11,11 +11,11 @@ import "grpc/fscommon.proto"; import "proto/journal/file.proto"; enum WritePType { - MUST_CACHE = 1; - TRY_CACHE = 2; + MUST_CACHE = 1 [deprecated = true]; + TRY_CACHE = 2 [deprecated = true]; CACHE_THROUGH = 3; THROUGH = 4; - ASYNC_THROUGH = 5; + ASYNC_THROUGH = 5 [deprecated = true]; NONE = 6; } @@ -117,7 +117,7 @@ message CreateDirectoryPOptions { optional bool recursive = 1; optional bool allowExists = 2; optional PMode mode = 3; - optional WritePType writeType = 4; + optional WritePType writeType = 4 [default = CACHE_THROUGH]; optional FileSystemMasterCommonPOptions commonOptions = 5; map xattr = 6; optional XAttrPropagationStrategy xattrPropStrat = 7 [default = NEW_PATHS]; @@ -142,7 +142,7 @@ message CreateFilePOptions { optional int32 replicationMin = 5; optional int32 replicationDurable = 6; optional int32 writeTier = 7; - optional WritePType writeType = 8; + optional WritePType writeType = 8 [default = CACHE_THROUGH]; optional FileSystemMasterCommonPOptions commonOptions = 9; optional int64 persistenceWaitTime = 10; map xattr = 11; @@ -614,7 +614,7 @@ message CopyJobPOptions { optional bool verify = 2; optional bool partialListing = 3; optional bool overwrite = 4; - optional WritePType writeType = 5; + optional WritePType writeType = 5 [default = CACHE_THROUGH]; optional bool check_content = 6; } @@ -623,7 +623,7 @@ message MoveJobPOptions { optional bool verify = 2; optional bool partialListing = 3; optional bool overwrite = 4; - optional WritePType writeType = 5; + optional WritePType writeType = 5 [default = CACHE_THROUGH]; optional bool check_content = 6; } diff --git a/dora/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerDataWriter.java b/dora/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerDataWriter.java index ea1302e604bd..af0d80e3e4a5 100644 --- a/dora/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerDataWriter.java +++ b/dora/core/client/fs/src/main/java/alluxio/client/block/stream/BlockWorkerDataWriter.java @@ -11,7 +11,6 @@ package alluxio.client.block.stream; -import alluxio.client.WriteType; import alluxio.client.file.FileSystemContext; import alluxio.client.file.options.OutStreamOptions; import alluxio.conf.AlluxioConfiguration; @@ -44,7 +43,6 @@ public final class BlockWorkerDataWriter implements DataWriter { private final BlockWriter mBlockWriter; private final BlockWorker mBlockWorker; private final int mChunkSize; - private final OutStreamOptions mOptions; private final long mSessionId; private final long mBufferSize; private final long mReservedBytes; @@ -132,8 +130,7 @@ public void flush() {} public void close() throws IOException { mBlockWriter.close(); try { - mBlockWorker.commitBlock(mSessionId, mBlockId, - mOptions.getWriteType() == WriteType.ASYNC_THROUGH); + mBlockWorker.commitBlock(mSessionId, mBlockId, false); } catch (Exception e) { mBlockWorker.cleanupSession(mSessionId); throw new IOException(e); @@ -157,7 +154,6 @@ private BlockWorkerDataWriter(long sessionId, long blockId, OutStreamOptions opt mBlockWriter = blockWriter; mChunkSize = chunkSize; mBlockId = blockId; - mOptions = options; mSessionId = sessionId; mReservedBytes = reservedBytes; mBufferSize = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES); diff --git a/dora/core/client/fs/src/main/java/alluxio/client/block/stream/GrpcDataWriter.java b/dora/core/client/fs/src/main/java/alluxio/client/block/stream/GrpcDataWriter.java index 569b2be9d8ee..80c272ec1c5e 100644 --- a/dora/core/client/fs/src/main/java/alluxio/client/block/stream/GrpcDataWriter.java +++ b/dora/core/client/fs/src/main/java/alluxio/client/block/stream/GrpcDataWriter.java @@ -11,7 +11,6 @@ package alluxio.client.block.stream; -import alluxio.client.WriteType; import alluxio.client.file.FileSystemContext; import alluxio.client.file.options.OutStreamOptions; import alluxio.conf.AlluxioConfiguration; @@ -135,7 +134,6 @@ private GrpcDataWriter(FileSystemContext context, final WorkerNetAddress address builder.setCreateUfsFileOptions(ufsFileOptions); } // check if we need to pin block on create - builder.setPinOnCreate(options.getWriteType() == WriteType.ASYNC_THROUGH); builder.setSpaceToReserve(reservedBytes); mPartialRequest = builder.buildPartial(); mChunkSize = chunkSize; diff --git a/dora/core/client/fs/src/test/java/alluxio/client/file/FileSystemTestUtils.java b/dora/core/client/fs/src/test/java/alluxio/client/file/FileSystemTestUtils.java index 7afdbbb5c672..43df3b508e8e 100644 --- a/dora/core/client/fs/src/test/java/alluxio/client/file/FileSystemTestUtils.java +++ b/dora/core/client/fs/src/test/java/alluxio/client/file/FileSystemTestUtils.java @@ -32,6 +32,8 @@ */ @ThreadSafe public final class FileSystemTestUtils { + public static final WritePType DEFAULT_WRITE_TYPE = WritePType.CACHE_THROUGH; + /** * Creates a simple file with {@code len} bytes. * @@ -45,6 +47,17 @@ public static void createByteFile(FileSystem fs, String fileName, int len, createByteFile(fs, new AlluxioURI(fileName), options, len); } + /** + * Creates a simple file with {@code len} bytes. + * + * @param fs a {@link FileSystem} handler + * @param fileName the name of the file to be created + * @param len file size + */ + public static void createByteFile(FileSystem fs, String fileName, int len) { + createByteFile(fs, new AlluxioURI(fileName), DEFAULT_WRITE_TYPE, len); + } + /** * Creates a simple file with {@code len} bytes. * @@ -58,6 +71,17 @@ public static void createByteFile(FileSystem fs, String fileName, createByteFile(fs, new AlluxioURI(fileName), writeType, len); } + /** + * Creates a simple file with {@code len} bytes. + * + * @param fs a {@link FileSystem} handler + * @param fileURI URI of the file + * @param len file size + */ + public static void createByteFile(FileSystem fs, AlluxioURI fileURI, int len) { + createByteFile(fs, fileURI, DEFAULT_WRITE_TYPE, len); + } + /** * Creates a simple file with {@code len} bytes. * diff --git a/dora/core/common/src/main/java/alluxio/client/UnderStorageType.java b/dora/core/common/src/main/java/alluxio/client/UnderStorageType.java index ffc4e0f943b7..40f68e6932bc 100644 --- a/dora/core/common/src/main/java/alluxio/client/UnderStorageType.java +++ b/dora/core/common/src/main/java/alluxio/client/UnderStorageType.java @@ -25,9 +25,6 @@ public enum UnderStorageType { /** Do not persist data to the under storage. */ NO_PERSIST(2), - - /** Persist data to the under storage asynchronously. */ - ASYNC_PERSIST(3), ; private final int mValue; @@ -42,11 +39,4 @@ public enum UnderStorageType { public boolean isSyncPersist() { return mValue == SYNC_PERSIST.mValue; } - - /** - * @return whether the data should be persisted to the under storage asynchronously - */ - public boolean isAsyncPersist() { - return mValue == ASYNC_PERSIST.mValue; - } } diff --git a/dora/core/common/src/main/java/alluxio/client/WriteType.java b/dora/core/common/src/main/java/alluxio/client/WriteType.java index 10476f146da3..1337191dd147 100644 --- a/dora/core/common/src/main/java/alluxio/client/WriteType.java +++ b/dora/core/common/src/main/java/alluxio/client/WriteType.java @@ -26,13 +26,16 @@ public enum WriteType { * Write the file, guaranteeing the data is written to Alluxio storage or failing the operation. * The data will be written to the highest tier in a worker's storage. Data will not be * persisted to the under storage. + * + * @deprecated This write type is deprecated as of v3.0 and not recommended for use. */ + @Deprecated MUST_CACHE(1), /** * Write the file and try to cache it. * * @deprecated This write type is deprecated as of v0.8 and not recommended for use. Use either - * {@link #MUST_CACHE} or {@link #CACHE_THROUGH} depending on your data persistence + * {@link #CACHE_THROUGH} or {@link #THROUGH} depending on your data persistence * requirements. */ @Deprecated @@ -48,7 +51,10 @@ public enum WriteType { THROUGH(4), /** * [Experimental] Write the file asynchronously to the under fs. + * + * @deprecated This write type is deprecated as of v3.0 and not recommended for use. */ + @Deprecated ASYNC_THROUGH(5), /** * Do not store the data in Alluxio or Under Storage. This write type should only be used for @@ -79,8 +85,6 @@ public AlluxioStorageType getAlluxioStorageType() { public UnderStorageType getUnderStorageType() { if (isThrough()) { return UnderStorageType.SYNC_PERSIST; - } else if (isAsync()) { - return UnderStorageType.ASYNC_PERSIST; } return UnderStorageType.NO_PERSIST; } @@ -92,22 +96,12 @@ public int getValue() { return mValue; } - /** - * @return true if by this write type data will be persisted asynchronously to under - * storage (e.g., {@link #ASYNC_THROUGH}), false otherwise - */ - public boolean isAsync() { - return mValue == ASYNC_THROUGH.mValue; - } - /** * @return true if by this write type data will be cached in Alluxio space (e.g., - * {@link #MUST_CACHE}, {@link #CACHE_THROUGH}, {@link #TRY_CACHE}, or - * {@link #ASYNC_THROUGH}), false otherwise + * {@link #CACHE_THROUGH}, {@link #TRY_CACHE}, false otherwise */ public boolean isCache() { - return (mValue == MUST_CACHE.mValue) || (mValue == CACHE_THROUGH.mValue) - || (mValue == TRY_CACHE.mValue) || (mValue == ASYNC_THROUGH.mValue); + return (mValue == CACHE_THROUGH.mValue) || (mValue == TRY_CACHE.mValue); } /** diff --git a/dora/core/server/master/src/main/java/alluxio/master/file/meta/InodeTree.java b/dora/core/server/master/src/main/java/alluxio/master/file/meta/InodeTree.java index 13ccf4241635..9ac9d9dcebf7 100644 --- a/dora/core/server/master/src/main/java/alluxio/master/file/meta/InodeTree.java +++ b/dora/core/server/master/src/main/java/alluxio/master/file/meta/InodeTree.java @@ -12,7 +12,6 @@ package alluxio.master.file.meta; import alluxio.AlluxioURI; -import alluxio.client.WriteType; import alluxio.collections.Pair; import alluxio.concurrent.LockMode; import alluxio.conf.Configuration; @@ -1034,9 +1033,6 @@ public List createPath(RpcContext rpcContext, LockedInodePath inodePath, if (fileContext.isCacheable()) { newFile.setCacheable(true); } - if (fileContext.getWriteType() == WriteType.ASYNC_THROUGH) { - newFile.setPersistenceState(PersistenceState.TO_BE_PERSISTED); - } if (context.getXAttr() != null) { newFile.setXAttr(context.getXAttr()); } diff --git a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterFsOptsTest.java b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterFsOptsTest.java index b58219b7f135..a265a4c98ccd 100644 --- a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterFsOptsTest.java +++ b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterFsOptsTest.java @@ -22,7 +22,7 @@ import alluxio.AuthenticatedClientUserResource; import alluxio.AuthenticatedUserRule; import alluxio.Constants; -import alluxio.client.WriteType; +import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.conf.Configuration; import alluxio.exception.AccessControlException; import alluxio.exception.BlockInfoException; @@ -63,12 +63,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.io.Closeable; -import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -78,6 +78,9 @@ import java.util.Set; import java.util.stream.Collectors; +@DoraTestTodoItem(action = DoraTestTodoItem.Action.REMOVE, owner = "jiacheng", + comment = "remove FileSystemMaster unit testcases") +@Ignore @RunWith(Parameterized.class) public class FileSystemMasterFsOptsTest extends FileSystemMasterTestBase { @@ -85,18 +88,6 @@ public FileSystemMasterFsOptsTest(InodeStore.Factory factory) { mInodeStoreFactory = factory; } - @Test - public void createFileMustCacheThenCacheThrough() throws Exception { - File file = mTestFolder.newFile(); - AlluxioURI path = new AlluxioURI("/test"); - mFileSystemMaster.createFile(path, - CreateFileContext.defaults().setWriteType(WriteType.MUST_CACHE)); - - mThrown.expect(FileAlreadyExistsException.class); - mFileSystemMaster.createFile(path, - CreateFileContext.defaults().setWriteType(WriteType.MUST_CACHE)); - } - @Test public void createFileUsesOperationTime() throws Exception { AlluxioURI path = new AlluxioURI("/test"); diff --git a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTest.java b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTest.java index 9c41577513a9..bf1a36d2eff5 100644 --- a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTest.java +++ b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTest.java @@ -55,7 +55,6 @@ import alluxio.grpc.XAttrPropagationStrategy; import alluxio.heartbeat.HeartbeatContext; import alluxio.heartbeat.HeartbeatScheduler; -import alluxio.master.file.contexts.CompleteFileContext; import alluxio.master.file.contexts.CreateDirectoryContext; import alluxio.master.file.contexts.CreateFileContext; import alluxio.master.file.contexts.DeleteContext; @@ -89,6 +88,7 @@ import com.google.common.math.IntMath; import com.google.protobuf.ByteString; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -483,8 +483,10 @@ public void setAclWithoutOwner() throws Exception { } @Test + @Ignore public void setAclNestedWithoutOwner() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); + // not supported to set owner with non-existed user in local ufs mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.mergeFrom(SetAttributePOptions .newBuilder().setMode(new Mode((short) 0777).toProto()).setOwner("userA"))); Set entries = Sets.newHashSet(mFileSystemMaster @@ -994,32 +996,6 @@ public void permission() throws Exception { mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getMode()); } - /** - * Tests that a file is fully written to memory. - */ - @Test - public void isFullyInMemory() throws Exception { - // add nested file - mFileSystemMaster.createFile(NESTED_FILE_URI, mNestedFileContext); - // add in-memory block - long blockId = mFileSystemMaster.getNewBlockIdForFile(NESTED_FILE_URI); - mBlockMaster.commitBlock(mWorkerId1, Constants.KB, - Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, Constants.KB); - // add SSD block - blockId = mFileSystemMaster.getNewBlockIdForFile(NESTED_FILE_URI); - mBlockMaster.commitBlock(mWorkerId1, Constants.KB, Constants.MEDIUM_SSD, - Constants.MEDIUM_SSD, blockId, Constants.KB); - mFileSystemMaster.completeFile(NESTED_FILE_URI, CompleteFileContext.defaults()); - - // Create 2 files in memory. - createFileWithSingleBlock(ROOT_FILE_URI); - AlluxioURI nestedMemUri = NESTED_URI.join("mem_file"); - createFileWithSingleBlock(nestedMemUri); - assertEquals(2, mFileSystemMaster.getInMemoryFiles().size()); - assertTrue(mFileSystemMaster.getInMemoryFiles().contains(ROOT_FILE_URI)); - assertTrue(mFileSystemMaster.getInMemoryFiles().contains(nestedMemUri)); - } - /** * Tests {@link FileSystemMaster#free} on persisted file. */ @@ -1045,6 +1021,7 @@ public void free() throws Exception { * Tests {@link FileSystemMaster#free} on non-persisted file. */ @Test + @Ignore public void freeNonPersistedFile() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); mThrown.expect(UnexpectedAlluxioException.class); @@ -1133,6 +1110,7 @@ public void freeDir() throws Exception { * Tests the {@link FileSystemMaster#free} method with a directory with a file non-persisted. */ @Test + @Ignore public void freeDirWithNonPersistedFile() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); mThrown.expect(UnexpectedAlluxioException.class); @@ -1383,7 +1361,7 @@ public void unmountNonMountPointWithException() throws Exception { mFileSystemMaster.mount(alluxioURI, ufsURI, MountContext.defaults()); AlluxioURI dirURI = alluxioURI.join("dir"); mFileSystemMaster.createDirectory(dirURI, CreateDirectoryContext - .defaults().setWriteType(WriteType.MUST_CACHE)); + .defaults().setWriteType(WriteType.CACHE_THROUGH)); mThrown.expect(InvalidPathException.class); mFileSystemMaster.unmount(dirURI); } @@ -1428,6 +1406,7 @@ public void workerHeartbeat() throws Exception { * Tests that lost files can successfully be detected. */ @Test + @Ignore public void lostFilesDetection() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); long fileId = mFileSystemMaster.getFileId(NESTED_FILE_URI); @@ -1498,6 +1477,7 @@ public void ignoreInvalidFiles() throws Exception { } @Test + @Ignore public void propagatePersisted() throws Exception { AlluxioURI nestedFile = new AlluxioURI("/nested1/nested2/file"); AlluxioURI parent1 = new AlluxioURI("/nested1/"); diff --git a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTestBase.java b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTestBase.java index 471dd7aba0d0..cb064a7f5d80 100644 --- a/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTestBase.java +++ b/dora/core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTestBase.java @@ -196,7 +196,6 @@ public void before() throws Exception { mUnderFS = Configuration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); mNestedFileContext = CreateFileContext.mergeFrom( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) - .setWriteType(WritePType.MUST_CACHE) .setRecursive(true)); mJournalFolder = mTestFolder.newFolder().getAbsolutePath(); startServices(); @@ -331,8 +330,7 @@ public long createFileWithSingleBlock(AlluxioURI uri, CreateFileContext createFi // write the file WritePType writeType = createFileContext.getOptions().getWriteType(); if (info.getUfsPath() != null && ( - writeType == WritePType.CACHE_THROUGH || writeType == WritePType.THROUGH - || writeType == WritePType.ASYNC_THROUGH)) { + writeType == WritePType.CACHE_THROUGH || writeType == WritePType.THROUGH)) { Files.createFile(Paths.get(info.getUfsPath())); } mBlockMaster.commitBlock(mWorkerId1, Constants.KB, diff --git a/dora/core/server/master/src/test/java/alluxio/master/journal/sink/JournalSinkTest.java b/dora/core/server/master/src/test/java/alluxio/master/journal/sink/JournalSinkTest.java index da37ca87d6ca..b31da1899649 100644 --- a/dora/core/server/master/src/test/java/alluxio/master/journal/sink/JournalSinkTest.java +++ b/dora/core/server/master/src/test/java/alluxio/master/journal/sink/JournalSinkTest.java @@ -18,6 +18,7 @@ import alluxio.AlluxioTestDirectory; import alluxio.AlluxioURI; import alluxio.ConfigurationRule; +import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; import alluxio.grpc.CreateDirectoryPOptions; @@ -45,6 +46,7 @@ import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -61,6 +63,9 @@ /** * Tests for {@link JournalSinkTest}. */ +@Ignore +@DoraTestTodoItem(action = DoraTestTodoItem.Action.REMOVE, owner = "Jiacheng", + comment = "journal no longer exists in dora") public final class JournalSinkTest { private static final Logger LOG = LoggerFactory.getLogger(JournalSinkTest.class); private static final long INVALID_ID = -1; diff --git a/dora/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java b/dora/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java index e9e623d4397d..e6c2b0f52a7e 100644 --- a/dora/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java +++ b/dora/core/server/worker/src/main/java/alluxio/worker/grpc/WriteRequest.java @@ -25,15 +25,11 @@ public class WriteRequest { /** This ID can either be block ID or temp UFS file ID. */ private final long mId; - /** Whether to pin block on create. */ - private final boolean mPinOnCreate; - /** The session id associated with all temporary resources of this request. */ private final long mSessionId; WriteRequest(alluxio.grpc.WriteRequest request) { mId = request.getCommand().getId(); - mPinOnCreate = request.getCommand().getPinOnCreate(); mSessionId = IdUtils.createSessionId(); } @@ -44,13 +40,6 @@ public long getId() { return mId; } - /** - * @return whether to pin block on create - */ - public boolean getPinOnCreate() { - return mPinOnCreate; - } - /** * @return the session ID */ diff --git a/dora/job/server/src/test/java/alluxio/job/plan/migrate/MigrateDefinitionRunTaskTest.java b/dora/job/server/src/test/java/alluxio/job/plan/migrate/MigrateDefinitionRunTaskTest.java index f7a2fdea7087..a278eafcaf05 100644 --- a/dora/job/server/src/test/java/alluxio/job/plan/migrate/MigrateDefinitionRunTaskTest.java +++ b/dora/job/server/src/test/java/alluxio/job/plan/migrate/MigrateDefinitionRunTaskTest.java @@ -137,24 +137,6 @@ public void writeTypeTest() throws Exception { runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.CACHE_THROUGH); verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), ArgumentMatchers .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build())); - - runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.MUST_CACHE); - verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), ArgumentMatchers - .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build())); - } - - /** - * Tests the edge case where performing an WriteType.AsyncThrough with deleteSource of a - * persisted file writes the move synchronously. - */ - @Test - public void writeTypeAsyncThroughPersistedTest() throws Exception { - FileInfo fileInfo = new FileInfo(); - fileInfo.setPersisted(true); - when(mMockFileSystem.getStatus(eq(new AlluxioURI(TEST_SOURCE)))) - .thenReturn(new URIStatus(fileInfo)); - - runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.ASYNC_THROUGH); } @Test diff --git a/dora/shell/src/main/java/alluxio/cli/TestRunner.java b/dora/shell/src/main/java/alluxio/cli/TestRunner.java index 7dfd74ae61c7..c7b80c6e2d5a 100644 --- a/dora/shell/src/main/java/alluxio/cli/TestRunner.java +++ b/dora/shell/src/main/java/alluxio/cli/TestRunner.java @@ -46,8 +46,8 @@ public final class TestRunner { Arrays.asList(ReadType.CACHE_PROMOTE, ReadType.CACHE, ReadType.NO_CACHE); /** Write types to test. */ - private static final List WRITE_TYPES = Arrays.asList(WriteType.MUST_CACHE, - WriteType.CACHE_THROUGH, WriteType.THROUGH, WriteType.ASYNC_THROUGH); + private static final List WRITE_TYPES = + Arrays.asList(WriteType.CACHE_THROUGH, WriteType.THROUGH); @Parameter(names = "--directory", description = "Alluxio path for the tests working directory.") diff --git a/dora/tests/integration/src/main/java/alluxio/master/backcompat/BackwardsCompatibilityJournalGenerator.java b/dora/tests/integration/src/main/java/alluxio/master/backcompat/BackwardsCompatibilityJournalGenerator.java index ea2d15a95ea3..1d14a28a999b 100644 --- a/dora/tests/integration/src/main/java/alluxio/master/backcompat/BackwardsCompatibilityJournalGenerator.java +++ b/dora/tests/integration/src/main/java/alluxio/master/backcompat/BackwardsCompatibilityJournalGenerator.java @@ -17,7 +17,6 @@ import alluxio.conf.PropertyKey; import alluxio.grpc.BackupPOptions; import alluxio.grpc.BackupPRequest; -import alluxio.master.backcompat.ops.AsyncPersist; import alluxio.master.backcompat.ops.CreateDirectory; import alluxio.master.backcompat.ops.CreateFile; import alluxio.master.backcompat.ops.Delete; @@ -61,7 +60,6 @@ public final class BackwardsCompatibilityJournalGenerator { .add(new CreateDirectory(), new CreateFile(), new Mount(), - new AsyncPersist(), new Delete(), new PersistFile(), new PersistDirectory(), diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/JournalToolTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/JournalToolTest.java index 3cda389f7cdd..8561708e2af0 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/JournalToolTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/JournalToolTest.java @@ -83,7 +83,7 @@ public class JournalToolTest extends BaseIntegrationTest { .setProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS) .setProperty(PropertyKey.MASTER_JOURNAL_CHECKPOINT_PERIOD_ENTRIES, CHECKPOINT_SIZE) .setProperty(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX, "100") - .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE).build(); + .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH).build(); private File mDumpDir; private FileSystem mFs; diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/AbstractFileSystemShellTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/AbstractFileSystemShellTest.java index 998e979e0f43..f306ce63d62f 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/AbstractFileSystemShellTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/AbstractFileSystemShellTest.java @@ -27,7 +27,6 @@ import alluxio.grpc.LoadMetadataPType; import alluxio.grpc.OpenFilePOptions; import alluxio.grpc.ReadPType; -import alluxio.grpc.WritePType; import alluxio.master.LocalAlluxioCluster; import alluxio.master.LocalAlluxioJobCluster; import alluxio.master.job.JobMaster; @@ -142,8 +141,7 @@ public static void afterClass() throws Exception { * @param bytes file size */ protected void copyToLocalWithBytes(int bytes) throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, - bytes); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", bytes); sFsShell.run("copyToLocal", "/testFile", sLocalAlluxioCluster.getAlluxioHome() + "/testFile"); assertEquals(getCommandOutput(new String[] {"copyToLocal", "/testFile", diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/FileSystemShellUtilsTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/FileSystemShellUtilsTest.java index 96b727f6d17c..06beee4d3279 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/FileSystemShellUtilsTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/FileSystemShellUtilsTest.java @@ -101,7 +101,7 @@ public String resetFileHierarchy() throws IOException, AlluxioException { * @return the test directory */ public static String resetFileHierarchy(FileSystem fs) throws IOException, AlluxioException { - return resetFileHierarchy(fs, WritePType.MUST_CACHE); + return resetFileHierarchy(fs, WritePType.CACHE_THROUGH); } /** diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CatCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CatCommandIntegrationTest.java index b3283adde4af..3cf5376bf3d1 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CatCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CatCommandIntegrationTest.java @@ -60,7 +60,7 @@ public void catNotExist() throws Exception { @Test public void cat() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("cat", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(10); Assert.assertArrayEquals(expect, mOutput.toByteArray()); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChecksumCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChecksumCommandIntegrationTest.java index 6050cbecd13c..624aa217bdb9 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChecksumCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChecksumCommandIntegrationTest.java @@ -15,7 +15,6 @@ import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.file.FileSystemTestUtils; import alluxio.exception.ExceptionMessage; -import alluxio.grpc.WritePType; import alluxio.util.io.BufferUtils; import org.apache.commons.codec.digest.DigestUtils; @@ -36,7 +35,7 @@ public final class ChecksumCommandIntegrationTest extends AbstractFileSystemShel */ @Test public void checksum() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("checksum", "/testFile"); String str = mOutput.toString(); String[] splitString = str.split("\\s+"); @@ -55,10 +54,8 @@ public void checksum() throws Exception { */ @Test public void checksumWildCard() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFileA", - WritePType.MUST_CACHE, 10); - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFileB", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFileA", 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFileB", 10); sFsShell.run("checksum", "/testDir*/testFile*"); String str = mOutput.toString(); String[] splitString = str.split("\\s+"); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChgrpCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChgrpCommandIntegrationTest.java index fc359a43a018..b276415155c1 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChgrpCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChgrpCommandIntegrationTest.java @@ -16,7 +16,6 @@ import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.file.FileSystemTestUtils; import alluxio.exception.AlluxioException; -import alluxio.grpc.WritePType; import org.junit.Assert; import org.junit.Ignore; @@ -33,7 +32,7 @@ public final class ChgrpCommandIntegrationTest extends AbstractFileSystemShellTest { @Test public void chgrp() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("chgrp", "group1", "/testFile"); String group = sFileSystem.getStatus(new AlluxioURI("/testFile")).getGroup(); Assert.assertEquals("group1", group); @@ -47,8 +46,7 @@ public void chgrp() throws IOException, AlluxioException { */ @Test public void chgrpRecursive() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", 10); // "chgrp -R group1 /testDir" should apply to both dir and child file sFsShell.run("chgrp", "-R", "group1", "/testDir"); String group = sFileSystem.getStatus(new AlluxioURI("/testDir")).getGroup(); @@ -66,12 +64,9 @@ public void chgrpRecursive() throws IOException, AlluxioException { */ @Test public void chgrpWildcard() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/foo/testFile1", - WritePType.MUST_CACHE, 10); - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/foo/testFile2", - WritePType.MUST_CACHE, 10); - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/bar/testFile3", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/foo/testFile1", 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/foo/testFile2", 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/bar/testFile3", 10); sFsShell.run("chgrp", "group1", "/testDir/*/testFile*"); String group = sFileSystem.getStatus(new AlluxioURI("/testDir/foo/testFile1")).getGroup(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChmodCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChmodCommandIntegrationTest.java index ce299b10701b..da39ca8d55c4 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChmodCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChmodCommandIntegrationTest.java @@ -16,7 +16,6 @@ import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.file.FileSystemTestUtils; import alluxio.exception.AlluxioException; -import alluxio.grpc.WritePType; import org.junit.Assert; import org.junit.Ignore; @@ -33,7 +32,7 @@ public final class ChmodCommandIntegrationTest extends AbstractFileSystemShellTest { @Test public void chmod() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("chmod", "777", "/testFile"); int permission = sFileSystem.getStatus(new AlluxioURI("/testFile")).getMode(); Assert.assertEquals((short) 0777, permission); @@ -47,8 +46,7 @@ public void chmod() throws IOException, AlluxioException { */ @Test public void chmodRecursively() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", 10); sFsShell.run("chmod", "-R", "777", "/testDir"); int permission = sFileSystem.getStatus(new AlluxioURI("/testDir")).getMode(); Assert.assertEquals((short) 0777, permission); @@ -61,7 +59,7 @@ public void chmodRecursively() throws IOException, AlluxioException { @Test public void chmodSymbolic() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("chmod", "a=rwx", "/testFile"); int permission = sFileSystem.getStatus(new AlluxioURI("/testFile")).getMode(); Assert.assertEquals((short) 0777, permission); @@ -75,10 +73,8 @@ public void chmodSymbolic() throws IOException, AlluxioException { */ @Test public void chmodWildCard() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile1", - WritePType.MUST_CACHE, 10); - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFile2", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile1", 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFile2", 10); sFsShell.run("chmod", "a=rwx", "/testDir*/testFile*"); int permission = sFileSystem.getStatus(new AlluxioURI("/testDir/testFile1")).getMode(); Assert.assertEquals((short) 0777, permission); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChownCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChownCommandIntegrationTest.java index c6738164f622..60ca5bec788f 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChownCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/ChownCommandIntegrationTest.java @@ -19,7 +19,6 @@ import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; -import alluxio.grpc.WritePType; import com.google.common.collect.ImmutableMap; import org.junit.Assert; @@ -44,7 +43,7 @@ public final class ChownCommandIntegrationTest extends AbstractFileSystemShellTe @Test public void chown() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("chown", TEST_USER_1.getUser(), "/testFile"); String owner = sFileSystem.getStatus(new AlluxioURI("/testFile")).getOwner(); Assert.assertEquals(TEST_USER_1.getUser(), owner); @@ -55,7 +54,7 @@ public void chown() throws IOException, AlluxioException { @Test public void chownValidOwnerValidGroupSuccess() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); String newOwner = TEST_USER_1.getUser(); String group = "staff"; String expectedCommandOutput = @@ -67,7 +66,7 @@ public void chownValidOwnerValidGroupSuccess() throws Exception { @Test public void chownValidOwnerValidGroupFail() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); String newOwner = TEST_USER_2.getUser(); String originalOwner = sFileSystem.getStatus(new AlluxioURI("/testFile")).getOwner(); String originalGroup = sFileSystem.getStatus(new AlluxioURI("/testFile")).getGroup(); @@ -81,7 +80,7 @@ public void chownValidOwnerValidGroupFail() throws Exception { @Test public void chownInvalidOwnerValidGroup() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); String nonexistUser = "nonexistuser"; String originalOwner = sFileSystem.getStatus(new AlluxioURI("/testFile")).getOwner(); String originalGroup = sFileSystem.getStatus(new AlluxioURI("/testFile")).getGroup(); @@ -95,7 +94,7 @@ public void chownInvalidOwnerValidGroup() throws Exception { @Test public void chownValidOwnerInvalidGroup() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); String newOwner = TEST_USER_1.getUser(); String nonexistGroup = "nonexistgroup"; String originalOwner = sFileSystem.getStatus(new AlluxioURI("/testFile")).getOwner(); @@ -110,7 +109,7 @@ public void chownValidOwnerInvalidGroup() throws Exception { @Test public void chownInvalidOwnerInvalidGroup() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); String nonexistUser = "nonexistuser"; String nonexistGroup = "nonexistgroup"; String originalOwner = sFileSystem.getStatus(new AlluxioURI("/testFile")).getOwner(); @@ -128,8 +127,7 @@ public void chownInvalidOwnerInvalidGroup() throws Exception { */ @Test public void chownRecursive() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile", 10); sFsShell.run("chown", "-R", TEST_USER_1.getUser(), "/testDir"); String owner = sFileSystem.getStatus(new AlluxioURI("/testDir/testFile")).getOwner(); Assert.assertEquals(TEST_USER_1.getUser(), owner); @@ -145,10 +143,8 @@ public void chownRecursive() throws IOException, AlluxioException { */ @Test public void chownWildcard() throws IOException, AlluxioException { - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile1", - WritePType.MUST_CACHE, 10); - FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFile2", - WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir/testFile1", 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testDir2/testFile2", 10); sFsShell.run("chown", "-R", TEST_USER_1.getUser(), "/*/testFile*"); String owner = sFileSystem.getStatus(new AlluxioURI("/testDir/testFile1")).getOwner(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyFromLocalCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyFromLocalCommandIntegrationTest.java index ead23017ae8d..8033cde909f0 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyFromLocalCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyFromLocalCommandIntegrationTest.java @@ -14,15 +14,12 @@ import static org.hamcrest.CoreMatchers.containsString; import alluxio.AlluxioURI; -import alluxio.ConfigurationRule; import alluxio.SystemPropertyRule; import alluxio.annotation.dora.DoraTestTodoItem; -import alluxio.client.WriteType; import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.cli.fs.FileSystemShellUtilsTest; import alluxio.client.file.FileInStream; import alluxio.client.file.URIStatus; -import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; import alluxio.exception.ExceptionMessage; @@ -268,23 +265,6 @@ public void copyFromLocal() throws IOException, AlluxioException { Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read)); } - @Test - public void copyFromLocalMustCacheThenCacheThrough() throws Exception { - File file = mTestFolder.newFile(); - try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, - WriteType.MUST_CACHE.toString(), Configuration.modifiableGlobal()).toResource()) { - Assert.assertEquals(0, sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/test")); - } - try (Closeable c = new ConfigurationRule(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, - WriteType.CACHE_THROUGH.toString(), Configuration.modifiableGlobal()).toResource()) { - mOutput.reset(); - sFsShell.run("copyFromLocal", file.getAbsolutePath(), "/test"); - } - Assert.assertThat(mOutput.toString(), - containsString( - ExceptionMessage.CANNOT_OVERWRITE_FILE_WITHOUT_OVERWRITE.getMessage("/test"))); - } - @Test public void copyFromLocalTestWithFullURI() throws IOException, AlluxioException { File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10)); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyToLocalCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyToLocalCommandIntegrationTest.java index c9fb81efa765..596817879cd0 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyToLocalCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CopyToLocalCommandIntegrationTest.java @@ -16,7 +16,6 @@ import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.cli.fs.FileSystemShellUtilsTest; import alluxio.client.file.FileSystemTestUtils; -import alluxio.grpc.WritePType; import org.junit.Assert; import org.junit.Ignore; @@ -59,7 +58,7 @@ public void copyToLocalRelativePathDir() throws Exception { return; } - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); HashMap sysProps = new HashMap<>(); sysProps.put("user.dir", mTestFolder.getRoot().getAbsolutePath()); try (Closeable p = new SystemPropertyRule(sysProps).toResource()) { @@ -72,6 +71,7 @@ public void copyToLocalRelativePathDir() throws Exception { } @Test + @Ignore public void copyToLocalLarge() throws Exception { // Divide by 2 to avoid issues with async eviction. copyToLocalWithBytes(SIZE_BYTES / 2); @@ -141,7 +141,7 @@ public void copyToLocalRelativePath() throws Exception { HashMap sysProps = new HashMap<>(); sysProps.put("user.dir", mTestFolder.getRoot().getAbsolutePath()); try (Closeable p = new SystemPropertyRule(sysProps).toResource()) { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("copyToLocal", "/testFile", "."); Assert.assertEquals("Copied /testFile to file://" + mTestFolder.getRoot().getAbsolutePath() + "/testFile" + "\n", mOutput.toString()); @@ -154,8 +154,7 @@ public void copyToLocalRelativePath() throws Exception { @Test public void parseOption() { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, - 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); int ret = sFsShell.run("copyToLocal", "--buffersize", "1024", "/testFile", sLocalAlluxioCluster.getAlluxioHome() + "/testFile"); Assert.assertEquals(0, ret); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CpCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CpCommandIntegrationTest.java index f92b4be47ad7..b7a6d47687ce 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CpCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/CpCommandIntegrationTest.java @@ -33,7 +33,6 @@ import alluxio.grpc.SetAclAction; import alluxio.grpc.SetAclPOptions; import alluxio.grpc.SetAttributePOptions; -import alluxio.grpc.WritePType; import alluxio.security.authorization.AclAction; import alluxio.security.authorization.AclEntry; import alluxio.security.authorization.AclEntryType; @@ -674,8 +673,7 @@ public void copyToLocalWildcard() throws Exception { @Override protected void copyToLocalWithBytes(int bytes) throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, - bytes); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", bytes); String[] cmd = new String[] {"cp", "/testFile", "file://" + sLocalAlluxioCluster.getAlluxioHome() + "/testFile"}; sFsShell.run(cmd); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/FreeCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/FreeCommandIntegrationTest.java index da5bda54476e..e72fa031dbe8 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/FreeCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/FreeCommandIntegrationTest.java @@ -34,15 +34,6 @@ comment = "fix free command") @Ignore public final class FreeCommandIntegrationTest extends AbstractFileSystemShellTest { - @Test - public void freeNonPersistedFile() throws IOException, AlluxioException { - String fileName = "/testFile"; - FileSystemTestUtils.createByteFile(sFileSystem, fileName, WritePType.MUST_CACHE, 10); - // freeing non persisted files is expected to fail - assertEquals(-1, sFsShell.run("free", fileName)); - assertTrue(isInMemoryTest(fileName)); - } - @Test public void freePinnedFile() throws IOException, AlluxioException { String fileName = "/testFile"; @@ -70,18 +61,6 @@ public void free() throws IOException, AlluxioException { assertFalse(isInMemoryTest(fileName)); } - @Test - public void freeWildCardNonPersistedFile() throws IOException, AlluxioException { - String testDir = - FileSystemShellUtilsTest.resetFileHierarchy(sFileSystem, WritePType.MUST_CACHE); - assertEquals(-1, sFsShell.run("free", testDir + "/foo/*")); - // freeing non persisted files is expected to fail - assertTrue(isInMemoryTest(testDir + "/foo/foobar1")); - assertTrue(isInMemoryTest(testDir + "/foo/foobar2")); - assertTrue(isInMemoryTest(testDir + "/bar/foobar3")); - assertTrue(isInMemoryTest(testDir + "/foobar4")); - } - @Test public void freeWildCardPinnedFile() throws IOException, AlluxioException { String testDir = diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/HeadCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/HeadCommandIntegrationTest.java index 4a846e237b92..ff0d8f5816d9 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/HeadCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/HeadCommandIntegrationTest.java @@ -15,7 +15,6 @@ import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.cli.fs.FileSystemShellUtilsTest; import alluxio.client.file.FileSystemTestUtils; -import alluxio.grpc.WritePType; import alluxio.util.io.BufferUtils; import org.junit.Assert; @@ -31,14 +30,14 @@ public final class HeadCommandIntegrationTest extends AbstractFileSystemShellTest { @Test public void headEmptyFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/emptyFile", WritePType.MUST_CACHE, 0); + FileSystemTestUtils.createByteFile(sFileSystem, "/emptyFile", 0); int ret = sFsShell.run("head", "/emptyFile"); Assert.assertEquals(0, ret); } @Test public void headLargeFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 2048); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 2048); sFsShell.run("head", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(1024, 1024); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -52,7 +51,7 @@ public void headNotExit() throws Exception { @Test public void headSmallFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); sFsShell.run("head", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(10); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -78,8 +77,7 @@ public void headWildcard() throws Exception { @Test public void headFileWithUserSpecifiedBytes() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, - 10000); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10000); sFsShell.run("head", "-c", "10000", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(0, 10000); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -87,8 +85,7 @@ public void headFileWithUserSpecifiedBytes() throws Exception { @Test public void headFileWithUserSpecifiedBytesWithUnit() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, - 10000); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10000); sFsShell.run("head", "-c", "2KB", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(0, 2048); Assert.assertArrayEquals(expect, mOutput.toByteArray()); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandIntegrationTest.java index 2c4aa173211d..b1a47caf731c 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandIntegrationTest.java @@ -59,9 +59,9 @@ private void createFiles(String user) throws Exception { .create(new TestUserState(user, Configuration.global()).getSubject(), Configuration.global())); } - FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 10); FileSystemTestUtils - .createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20); + .createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.CACHE_THROUGH, 20); FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30); } @@ -185,7 +185,7 @@ public void lsrNoAcl() throws Exception { @Test public void lsWithFormatSpecifierCharacter() throws IOException, AlluxioException { String fileName = "/localhost%2C61764%2C1476207067267..meta.1476207073442.meta"; - FileSystemTestUtils.createByteFile(sFileSystem, fileName, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, fileName, WritePType.CACHE_THROUGH, 10); sFsShell.run("ls", "/"); // CHECKSTYLE.OFF: LineLengthExceed - Improve readability checkOutput(" 10 NOT_PERSISTED .+ .+ 100% /localhost%2C61764%2C1476207067267..meta.1476207073442.meta"); @@ -198,11 +198,11 @@ public void lsWithFormatSpecifierCharacter() throws IOException, AlluxioExceptio @Test public void lsWithSortByPath() throws IOException, AlluxioException { FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100); + .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.CACHE_THROUGH, 100); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.CACHE_THROUGH, 10); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50); + .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 50); sFsShell.run("ls", "--sort", "path", "/testRoot"); checkOutput( " 50 NOT_PERSISTED .+ .+ 100% /testRoot/testFileA", @@ -218,9 +218,9 @@ public void lsWithSortByAccessTime() throws IOException, AlluxioException { String oldFileRecentlyAccessed = "/testRoot/testFileRecent"; String oldFileName = "/testRoot/testFile"; FileSystemTestUtils - .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.CACHE_THROUGH, 10); FileSystemTestUtils - .createByteFile(sFileSystem, oldFileName, WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, oldFileName, WritePType.CACHE_THROUGH, 10); FileSystemTestUtils.loadFile(sFileSystem, oldFileRecentlyAccessed); sFsShell.run("ls", "--sort", "lastAccessTime", "/testRoot"); @@ -237,9 +237,9 @@ public void lsWithSortByCreationTime() throws IOException, AlluxioException { String oldFileRecentlyAccessed = "/testRoot/testFileRecent"; String oldFileName = "/testRoot/testFile"; FileSystemTestUtils - .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, oldFileRecentlyAccessed, WritePType.CACHE_THROUGH, 10); FileSystemTestUtils - .createByteFile(sFileSystem, oldFileName, WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, oldFileName, WritePType.CACHE_THROUGH, 10); FileSystemTestUtils.loadFile(sFileSystem, oldFileRecentlyAccessed); sFsShell.run("ls", "--sort", "creationTime", "/testRoot"); @@ -254,11 +254,11 @@ public void lsWithSortByCreationTime() throws IOException, AlluxioException { @Test public void lsWithSortBySize() throws IOException, AlluxioException { FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50); + .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 50, 50); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10); + .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.CACHE_THROUGH, 10, 10); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100); + .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.CACHE_THROUGH, 100, 100); sFsShell.run("ls", "--sort", "size", "/testRoot"); checkOutput( " 10 NOT_PERSISTED .+ .+ 100% /testRoot/testFileZ", @@ -272,11 +272,11 @@ public void lsWithSortBySize() throws IOException, AlluxioException { @Test public void lsWithSortBySizeAndReverse() throws IOException, AlluxioException { FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50); + .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 50, 50); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10); + .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.CACHE_THROUGH, 10, 10); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100); + .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.CACHE_THROUGH, 100, 100); sFsShell.run("ls", "--sort", "size", "-r", "/testRoot"); checkOutput( " 100 NOT_PERSISTED .+ .+ 100% /testRoot/testLongFile", @@ -290,7 +290,7 @@ public void lsWithSortBySizeAndReverse() throws IOException, AlluxioException { @Test public void lsWithInvalidSortOption() throws IOException, AlluxioException { FileSystemTestUtils.createByteFile(sFileSystem, "/testRoot/testFileA", - WritePType.MUST_CACHE, 50, 50); + WritePType.CACHE_THROUGH, 50, 50); sFsShell.run("ls", "--sort", "unknownfield", "/testRoot"); String expected = "Invalid sort option `unknownfield` for --sort\n"; assertEquals(expected, mOutput.toString()); @@ -302,11 +302,11 @@ public void lsWithInvalidSortOption() throws IOException, AlluxioException { @Test public void lsReverseWithoutSort() throws IOException, AlluxioException { FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.MUST_CACHE, 50, 50); + .createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 50, 50); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.MUST_CACHE, 10, 10); + .createByteFile(sFileSystem, "/testRoot/testFileZ", WritePType.CACHE_THROUGH, 10, 10); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.MUST_CACHE, 100, 100); + .createByteFile(sFileSystem, "/testRoot/testLongFile", WritePType.CACHE_THROUGH, 100, 100); sFsShell.run("ls", "-r", "--sort", "path", "/testRoot"); checkOutput( " 100 NOT_PERSISTED .+ .+ 100% /testRoot/testLongFile", diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandSecurityIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandSecurityIntegrationTest.java index 577211f52812..5ad7e09cfc6a 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandSecurityIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/LsCommandSecurityIntegrationTest.java @@ -58,9 +58,9 @@ private void createFiles() throws Exception { FileSystem fs = sLocalAlluxioCluster.getClient(FileSystemContext .create(new TestUserState("test_user_ls", Configuration.global()).getSubject(), Configuration.global())); - FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.CACHE_THROUGH, 10); FileSystemTestUtils - .createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20); + .createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.CACHE_THROUGH, 20); FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30); } @@ -171,9 +171,9 @@ public void lsWithExtendedAcl() throws IOException, AlluxioException { .create(new TestUserState("test_user_ls", Configuration.global()).getSubject(), Configuration.global())); FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", - WritePType.MUST_CACHE, 20); + WritePType.CACHE_THROUGH, 20); FileSystemTestUtils.createByteFile(fs, "/testRoot/testFile", - WritePType.MUST_CACHE, size, size); + WritePType.CACHE_THROUGH, size, size); sFsShell.run("ls", "--sort", "path", "/testRoot"); // CHECKSTYLE.OFF: LineLengthExceed - Improve readability diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/OldLoadCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/OldLoadCommandIntegrationTest.java index 684f48689e18..9221bc468f0a 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/OldLoadCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/OldLoadCommandIntegrationTest.java @@ -38,7 +38,7 @@ public void loadDir() throws Exception { FileSystemTestUtils.createByteFile(sFileSystem, "/testRoot/testFileA", WritePType.THROUGH, 10); FileSystemTestUtils - .createByteFile(sFileSystem, "/testRoot/testFileB", WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, "/testRoot/testFileB", WritePType.CACHE_THROUGH, 10); AlluxioURI uriA = new AlluxioURI("/testRoot/testFileA"); AlluxioURI uriB = new AlluxioURI("/testRoot/testFileB"); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TailCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TailCommandIntegrationTest.java index e2472853f40d..e5828feaf3d7 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TailCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TailCommandIntegrationTest.java @@ -33,14 +33,14 @@ public final class TailCommandIntegrationTest extends AbstractFileSystemShellTest { @Test public void tailEmptyFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/emptyFile", WritePType.MUST_CACHE, 0); + FileSystemTestUtils.createByteFile(sFileSystem, "/emptyFile", WritePType.CACHE_THROUGH, 0); int ret = sFsShell.run("tail", "/emptyFile"); Assert.assertEquals(0, ret); } @Test public void tailLargeFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 2048); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.CACHE_THROUGH, 2048); sFsShell.run("tail", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(1024, 1024); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -54,7 +54,7 @@ public void tailNotExist() throws IOException { @Test public void tailSmallFile() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.CACHE_THROUGH, 10); sFsShell.run("tail", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(10); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -80,7 +80,7 @@ public void tailWildcard() throws Exception { @Test public void tailFileWithUserSpecifiedBytes() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 2048); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.CACHE_THROUGH, 2048); sFsShell.run("tail", "-c", "2000", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(48, 2000); Assert.assertArrayEquals(expect, mOutput.toByteArray()); @@ -88,7 +88,7 @@ public void tailFileWithUserSpecifiedBytes() throws Exception { @Test public void tailFileWithUserSpecifiedBytesWithUnit() throws Exception { - FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.CACHE_THROUGH, 10000); sFsShell.run("tail", "-c", "2KB", "/testFile"); byte[] expect = BufferUtils.getIncreasingByteArray(10000 - 2048, 2048); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TestCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TestCommandIntegrationTest.java index b210b3a1b151..07a52d926f31 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TestCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/TestCommandIntegrationTest.java @@ -15,7 +15,6 @@ import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.client.cli.fs.AbstractFileSystemShellTest; import alluxio.client.file.FileSystemTestUtils; -import alluxio.grpc.WritePType; import org.junit.Assert; import org.junit.Ignore; @@ -44,8 +43,7 @@ public void testPathIsDirectoryWhenPathIsDirectory() throws Exception { @Test @Ignore public void testPathIsDirectoryWhenPathIsFile() throws Exception { - FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); int ret = sFsShell.run("test", "-d", "/testFile"); Assert.assertEquals(1, ret); } @@ -66,8 +64,7 @@ public void testPathIsFileWhenPathIsDirectory() throws Exception { @Test @Ignore public void testPathIsFileWhenPathIsFile() throws Exception { - FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); int ret = sFsShell.run("test", "-f", "/testFile"); Assert.assertEquals(0, ret); } @@ -88,8 +85,7 @@ public void testPathExistWhenPathIsDirectory() throws Exception { @Test @Ignore public void testPathExistWhenPathIsFile() throws Exception { - FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", 10); int ret = sFsShell.run("test", "-e", "/testFile"); Assert.assertEquals(0, ret); } @@ -104,9 +100,9 @@ public void testDirNotEmptyWhenPathNotExist() throws Exception { @Ignore public void testDirNotEmptyWhenPathIsDirectory() throws Exception { FileSystemTestUtils - .createByteFile(sFileSystem, "/testDir1/testFile1", WritePType.MUST_CACHE, 0); + .createByteFile(sFileSystem, "/testDir1/testFile1", 0); FileSystemTestUtils - .createByteFile(sFileSystem, "/testDir2/testFile2", WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, "/testDir2/testFile2", 10); String[] command = new String[] {"mkdir", "/testDir3/testDir4"}; sFsShell.run(command); @@ -126,9 +122,9 @@ public void testDirNotEmptyWhenPathIsDirectory() throws Exception { @Test public void testDirNotEmptyWhenPathIsFile() throws Exception { FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile1", WritePType.MUST_CACHE, 0); + .createByteFile(sFileSystem, "/testFile1", 0); FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile2", WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, "/testFile2", 10); int ret = sFsShell.run("test", "-s", "/testFile1"); Assert.assertEquals(1, ret); @@ -153,9 +149,9 @@ public void testFileZeroLengthWhenPathIsDirectory() throws Exception { @Ignore public void testFileZeroLengthWhenPathIsFile() throws Exception { FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile1", WritePType.MUST_CACHE, 0); + .createByteFile(sFileSystem, "/testFile1", 0); FileSystemTestUtils - .createByteFile(sFileSystem, "/testFile2", WritePType.MUST_CACHE, 10); + .createByteFile(sFileSystem, "/testFile2", 10); int ret = sFsShell.run("test", "-z", "/testFile1"); Assert.assertEquals(0, ret); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/UnsetTtlTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/UnsetTtlTest.java index 3852906639b8..7aed1432c5f9 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/UnsetTtlTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fs/command/UnsetTtlTest.java @@ -34,7 +34,7 @@ public final class UnsetTtlTest extends AbstractFileSystemShellTest { public void unsetTtl() throws Exception { String filePath = "/testFile"; AlluxioURI uri = new AlluxioURI("/testFile"); - FileSystemTestUtils.createByteFile(sFileSystem, filePath, WritePType.MUST_CACHE, 1); + FileSystemTestUtils.createByteFile(sFileSystem, filePath, WritePType.CACHE_THROUGH, 1); assertEquals(Constants.NO_TTL, sFileSystem.getStatus(uri).getTtl()); // unsetTTL on a file originally with no TTL will leave the TTL unchanged. diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportCommandIntegrationTest.java index de7b8032719a..196b00349fc0 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportCommandIntegrationTest.java @@ -11,11 +11,13 @@ package alluxio.client.cli.fsadmin.command; +import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.cli.fsadmin.command.ReportCommand; import alluxio.client.cli.fsadmin.AbstractFsAdminShellTest; import org.hamcrest.CoreMatchers; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; /** @@ -23,6 +25,9 @@ */ public final class ReportCommandIntegrationTest extends AbstractFsAdminShellTest { @Test + @DoraTestTodoItem(owner = "jiacheng", action = DoraTestTodoItem.Action.FIX, + comment = "need to check if this command will still exist in Dora") + @Ignore public void masterNotRunning() throws Exception { mLocalAlluxioCluster.stopMasters(); int ret = mFsAdminShell.run("report"); diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportEmbeddedJournalIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportEmbeddedJournalIntegrationTest.java index 668e01d764e6..c8894b38aa3a 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportEmbeddedJournalIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/ReportEmbeddedJournalIntegrationTest.java @@ -29,6 +29,9 @@ PropertyKey.Name.MASTER_JOURNAL_TYPE, "EMBEDDED", PropertyKey.Name.MASTER_EMBEDDED_JOURNAL_PORT, ReportEmbeddedJournalIntegrationTest.JOURNAL_PORT}) +@Ignore +@DoraTestTodoItem(action = DoraTestTodoItem.Action.REMOVE, owner = "Jiacheng", + comment = "journal no longer exists in dora") public class ReportEmbeddedJournalIntegrationTest extends AbstractFsAdminShellTest { static final String JOURNAL_PORT = "0"; diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/metrics/MetricsCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/metrics/MetricsCommandIntegrationTest.java index 13d1739c5dfe..281282bfc7e2 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/metrics/MetricsCommandIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/metrics/MetricsCommandIntegrationTest.java @@ -35,7 +35,7 @@ public final class MetricsCommandIntegrationTest extends AbstractFsAdminShellTes @Test public void clearLeadingMasterMetrics() throws IOException { FileSystemTestUtils.createByteFile(mLocalAlluxioCluster.getClient(), - "/file", WritePType.MUST_CACHE, 10); + "/file", WritePType.CACHE_THROUGH, 10); int errCode = mFsAdminShell.run("report", "metrics"); assertEquals("", mErrOutput.toString()); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemIntegrationTest.java index 2dcd452c0175..19a9d66014d1 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemIntegrationTest.java @@ -346,7 +346,7 @@ public void getBlockLocations() throws Exception { @Test public void testMultiSetAttribute() throws Exception { AlluxioURI testFile = new AlluxioURI("/test1"); - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 512); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 512); long expectedTtl = Configuration.getMs(PropertyKey.USER_FILE_CREATE_TTL); URIStatus stat = mFileSystem.getStatus(testFile); assertEquals("TTL should be equal to configuration", expectedTtl, stat.getTtl()); @@ -385,7 +385,7 @@ public void testMultiSetAttribute() throws Exception { @Test public void testTtlActionSetAttribute() throws Exception { AlluxioURI testFile = new AlluxioURI("/test1"); - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 512); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 512); TtlAction expectedAction = Configuration.getEnum(PropertyKey.USER_FILE_CREATE_TTL_ACTION, TtlAction.class); URIStatus stat = mFileSystem.getStatus(testFile); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemMasterIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemMasterIntegrationTest.java index 24d29ca7cd7e..af52f7b94201 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemMasterIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemMasterIntegrationTest.java @@ -454,7 +454,7 @@ public void deleteDirectoryRecursive() throws Exception { FileSystem fs = sLocalAlluxioClusterResource.get().getClient(); for (int i = 0; i < 3; i++) { FileSystemTestUtils.createByteFile(fs, PathUtils.concatPath(dir, "file" + i), 100, - CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build()); + CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build()); } fs.delete(dir, DeletePOptions.newBuilder().setRecursive(true).build()); assertFalse(fs.exists(dir)); @@ -734,9 +734,6 @@ public void ufsModeCreateFile() throws Exception { mFsMaster.updateUfsMode(new AlluxioURI(mFsMaster.getUfsAddress()), UfsMode.READ_ONLY); - // Alluxio only should not be affected - mFsMaster.createFile(new AlluxioURI("/in_alluxio"), - CreateFileContext.defaults().setWriteType(WriteType.MUST_CACHE)); // Ufs file creation should throw an exception mThrown.expect(AccessControlException.class); mFsMaster.createFile(new AlluxioURI("/in_ufs"), @@ -748,9 +745,6 @@ public void ufsModeCreateDirectory() throws Exception { mFsMaster.updateUfsMode(new AlluxioURI(mFsMaster.getUfsAddress()), UfsMode.READ_ONLY); - // Alluxio only should not be affected - mFsMaster.createDirectory(new AlluxioURI("/in_alluxio"), - CreateDirectoryContext.defaults().setWriteType(WriteType.MUST_CACHE)); // Ufs file creation should throw an exception mThrown.expect(AccessControlException.class); mFsMaster.createDirectory(new AlluxioURI("/in_ufs"), diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemUtilsIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemUtilsIntegrationTest.java index 9e8b9e0e541c..e008f5decba5 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemUtilsIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/FileSystemUtilsIntegrationTest.java @@ -15,10 +15,8 @@ import static org.junit.Assert.assertTrue; import alluxio.AlluxioURI; -import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.client.file.FileOutStream; import alluxio.client.file.FileSystem; -import alluxio.client.file.FileSystemTestUtils; import alluxio.client.file.FileSystemUtils; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; @@ -34,14 +32,12 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.io.IOException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * Tests for {@link alluxio.client.file.FileSystemUtils}. @@ -183,30 +179,4 @@ public void run() { waitingThread.join(); writingThread.join(); } - - @Test - @Ignore - @DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "bowen", - comment = "check persist API") - public void waitPersistTimeoutTest() throws Exception { - String path = PathUtils.uniqPath(); - AlluxioURI alluxioPath = new AlluxioURI(path); - FileSystemTestUtils.createByteFile(sFileSystem, path, WritePType.MUST_CACHE, 4096); - assertFalse("File cannot yet be persisted", sFileSystem.getStatus(alluxioPath).isPersisted()); - mThrown.expect(TimeoutException.class); - FileSystemUtils.persistAndWait(sFileSystem, alluxioPath, 0, 1); // 1ms timeout - } - - @Test - @Ignore - @DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "bowen", - comment = "check persist API") - public void waitPersistIndefiniteTimeoutTest() throws Exception { - String path = PathUtils.uniqPath(); - AlluxioURI alluxioPath = new AlluxioURI(path); - FileSystemTestUtils.createByteFile(sFileSystem, path, WritePType.MUST_CACHE, 4096); - assertFalse("File cannot yet be persisted", sFileSystem.getStatus(alluxioPath).isPersisted()); - FileSystemUtils.persistAndWait(sFileSystem, alluxioPath, 5000, -1); - assertTrue("File must be persisted", sFileSystem.getStatus(alluxioPath).isPersisted()); - } } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/LoadMetadataIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/LoadMetadataIntegrationTest.java index c1b991aaa4b1..555351647600 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/LoadMetadataIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/LoadMetadataIntegrationTest.java @@ -115,7 +115,7 @@ public void before() throws Exception { fileWriter.write("test"); fileWriter.close(); FileOutStream stream = mFileSystem.createFile(new AlluxioURI("/mnt/mustcache/dir1/dir2/file1"), - CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(WritePType.MUST_CACHE) + CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(WritePType.CACHE_THROUGH) .build()); stream.write("test".getBytes(StandardCharsets.UTF_8)); stream.close(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/LocalFirstPolicyIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/LocalFirstPolicyIntegrationTest.java index 79928e6b400b..1d88411bf5a7 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/LocalFirstPolicyIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/LocalFirstPolicyIntegrationTest.java @@ -22,7 +22,6 @@ import alluxio.client.file.FileSystemTestUtils; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; -import alluxio.grpc.WritePType; import alluxio.master.AlluxioMasterProcess; import alluxio.master.TestUtils; import alluxio.network.TieredIdentityFactory; @@ -105,7 +104,7 @@ public void localNodeFirst() throws Exception { TieredIdentityFactory.fromString("node=node1,rack=rack1", Configuration.global())); try { - FileSystemTestUtils.createByteFile(fs, "/file1", WritePType.MUST_CACHE, 100); + FileSystemTestUtils.createByteFile(fs, "/file1", 100); } finally { Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null); } @@ -147,7 +146,7 @@ public void differentNodeOfSameRack() throws Exception { TieredIdentityFactory.fromString("node=node3,rack=rack2", Configuration.global())); try { - FileSystemTestUtils.createByteFile(fs, "/file2", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(fs, "/file2", 10); } finally { Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null); } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/PersistPermissionIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/PersistPermissionIntegrationTest.java index f82e876a0dbf..0c833a56a86b 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/PersistPermissionIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/PersistPermissionIntegrationTest.java @@ -21,9 +21,7 @@ import alluxio.grpc.CreateFilePOptions; import alluxio.grpc.WritePType; import alluxio.master.file.meta.PersistenceState; -import alluxio.testutils.IntegrationTestUtils; import alluxio.underfs.UnderFileSystem; -import alluxio.util.CommonUtils; import alluxio.util.UnderFileSystemUtils; import alluxio.util.io.PathUtils; @@ -77,64 +75,4 @@ public void syncPersistPermission() throws Exception { Assert.assertEquals(parentMode, mUfs.getDirectoryStatus(PathUtils.concatPath(mUfsRoot, filePath.getParent())).getMode()); } - - @Test - public void asyncPersistPermission() throws Exception { - // Skip non-local and non-HDFS UFSs. - Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs)); - - AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath()); - FileOutStream os = mFileSystem.createFile(filePath, CreateFilePOptions.newBuilder() - .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build()); - os.write((byte) 0); - os.write((byte) 1); - os.close(); - - CommonUtils.sleepMs(1); - // check the file is completed but not persisted - URIStatus status = mFileSystem.getStatus(filePath); - Assert.assertEquals(PersistenceState.TO_BE_PERSISTED.toString(), status.getPersistenceState()); - Assert.assertTrue(status.isCompleted()); - short fileMode = (short) status.getMode(); - short parentMode = (short) mFileSystem.getStatus(filePath.getParent()).getMode(); - - IntegrationTestUtils.waitForPersist(mLocalAlluxioClusterResource, filePath); - - status = mFileSystem.getStatus(filePath); - Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState()); - - // Check the permission of the created file and parent dir are in-sync between Alluxio and UFS. - Assert.assertEquals(fileMode, - mUfs.getFileStatus(PathUtils.concatPath(mUfsRoot, filePath)).getMode()); - Assert.assertEquals(parentMode, - mUfs.getDirectoryStatus(PathUtils.concatPath(mUfsRoot, filePath.getParent())).getMode()); - } - - @Test - public void asyncPersistEmptyFilePermission() throws Exception { - // Skip non-local and non-HDFS UFSs. - Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs)); - - AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath()); - mFileSystem.createFile(filePath, CreateFilePOptions.newBuilder() - .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build()).close(); - - // check the file is completed but not persisted - URIStatus status = mFileSystem.getStatus(filePath); - Assert.assertNotEquals(PersistenceState.PERSISTED, status.getPersistenceState()); - Assert.assertTrue(status.isCompleted()); - short fileMode = (short) status.getMode(); - short parentMode = (short) mFileSystem.getStatus(filePath.getParent()).getMode(); - - IntegrationTestUtils.waitForPersist(mLocalAlluxioClusterResource, filePath); - - status = mFileSystem.getStatus(filePath); - Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState()); - - // Check the permission of the created file and parent dir are in-sync between Alluxio and UFS. - Assert.assertEquals(fileMode, - mUfs.getFileStatus(PathUtils.concatPath(mUfsRoot, filePath)).getMode()); - Assert.assertEquals(parentMode, - mUfs.getDirectoryStatus(PathUtils.concatPath(mUfsRoot, filePath.getParent())).getMode()); - } } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/PinIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/PinIntegrationTest.java index 09d9f964bee6..457156b4a120 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/PinIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/PinIntegrationTest.java @@ -218,7 +218,7 @@ private String ufsPath(String path) { private void createEmptyFile(AlluxioURI fileURI) throws IOException, AlluxioException { CreateFilePOptions options = - CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build(); + CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).build(); FileOutStream os = mFileSystem.createFile(fileURI, options); os.close(); } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/RemoteReadIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/RemoteReadIntegrationTest.java index 1186591ce10b..4c3cdb9811ac 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/RemoteReadIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/RemoteReadIntegrationTest.java @@ -79,7 +79,7 @@ public final void before() throws Exception { mFileSystem = mLocalAlluxioClusterResource.get().getClient(); UserState us = UserState.Factory.create(Configuration.global()); mFsContext = FileSystemContext.create(us.getSubject(), Configuration.global()); - mWriteAlluxio = CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE) + mWriteAlluxio = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH) .setRecursive(true).build(); mWriteUnderStore = CreateFilePOptions.newBuilder().setWriteType(WritePType.THROUGH) .setRecursive(true).build(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/UfsSyncIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/UfsSyncIntegrationTest.java index 354dffdd57ae..380df4afb0ea 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/UfsSyncIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/UfsSyncIntegrationTest.java @@ -21,7 +21,6 @@ import alluxio.Constants; import alluxio.annotation.dora.DoraTestTodoItem; import alluxio.client.block.BlockMasterClient; -import alluxio.client.file.FileOutStream; import alluxio.client.file.FileSystem; import alluxio.client.file.FileSystemTestUtils; import alluxio.client.file.FileSystemUtils; @@ -303,33 +302,6 @@ public void renameFileSync() throws Exception { options); } - @Test - public void unpersistedFileSync() throws Exception { - ListStatusPOptions options = ListStatusPOptions.newBuilder() - .setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_ALWAYS).build(); - List initialStatusList = - mFileSystem.listStatus(new AlluxioURI(alluxioPath(ROOT_DIR)), options).stream() - .map(URIStatus::getName).collect(Collectors.toList()); - - // write a MUST_CACHE file - writeMustCacheFile(alluxioPath(NEW_FILE), 1); - - // List the status with force sync. - List syncStatusList = - mFileSystem.listStatus(new AlluxioURI(alluxioPath(ROOT_DIR)), options).stream() - .map(URIStatus::getName).collect(Collectors.toList()); - - Set initialSet = Sets.newHashSet(initialStatusList); - Set syncSet = Sets.newHashSet(syncStatusList); - assertTrue(syncSet.size() > initialSet.size()); - syncSet.removeAll(initialSet); - - // only the MUST_CACHE file should remain. - assertTrue(syncSet.size() == 1); - String file = syncSet.iterator().next(); - assertTrue(file.equals(new AlluxioURI(NEW_FILE).getName())); - } - @Test(timeout = 10000) public void lastModifiedLocalFileSync() throws Exception { int sizefactor = 10; @@ -909,16 +881,6 @@ private void writeUfsFile(String path, int sizeFactor) throws IOException { fileWriter.close(); } - private void writeMustCacheFile(String path, int sizeFactor) throws Exception { - CreateFilePOptions options = CreateFilePOptions.newBuilder() - .setWriteType(WritePType.MUST_CACHE).setRecursive(true).build(); - FileOutStream os = mFileSystem.createFile(new AlluxioURI(path), options); - for (int i = 0; i < sizeFactor; i++) { - os.write("test".getBytes()); - } - os.close(); - } - private void checkGetStatus(String path, GetStatusPOptions options, boolean expectExists) throws Exception { try { diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileInStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileInStreamIntegrationTest.java index 36738922b05a..f996f15b19bc 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileInStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileInStreamIntegrationTest.java @@ -67,7 +67,7 @@ public void FileInStreamConcurrency() throws Exception { Configuration.getInt(PropertyKey.USER_BLOCK_MASTER_CLIENT_POOL_SIZE_MAX) * 10; AlluxioURI uniqPath = new AlluxioURI(PathUtils.uniqPath()); FileSystemTestUtils.createByteFile(mFileSystem, uniqPath.getPath(), BLOCK_SIZE * 2, - CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).setRecursive(true) + CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true) .build()); List threads = new ArrayList<>(); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileSystemMasterCreateIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileSystemMasterCreateIntegrationTest.java index 835f1f4a64ed..ea72519f6902 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileSystemMasterCreateIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/concurrent/ConcurrentFileSystemMasterCreateIntegrationTest.java @@ -155,33 +155,6 @@ public void concurrentCreateExistingDir() throws Exception { } } - /** - * Test concurrent create of non-persisted directory. Directory is created as MUST_CACHE then - * files are created under that directory. - */ - @Test - public void concurrentCreateNonPersistedDir() throws Exception { - final int numThreads = CONCURRENCY_FACTOR; - // 7 nested components to create (2 seconds each). - final long limitMs = 14 * SLEEP_MS * CONCURRENCY_FACTOR / 2; - AlluxioURI[] paths = new AlluxioURI[numThreads]; - - // Create the existing path with MUST_CACHE, so subsequent creates have to persist the dirs. - mFileSystem.createDirectory(new AlluxioURI("/existing/path/dir/"), CreateDirectoryPOptions - .newBuilder().setRecursive(true).setWriteType(WritePType.MUST_CACHE).build()); - - for (int i = 0; i < numThreads; i++) { - paths[i] = - new AlluxioURI("/existing/path/dir/shared_dir/t_" + i + "/sub_dir1/sub_dir2/file" + i); - } - List errors = ConcurrentFileSystemMasterUtils - .unaryOperation(mFileSystem, ConcurrentFileSystemMasterUtils.UnaryOperation.CREATE, paths, - limitMs); - if (!errors.isEmpty()) { - Assert.fail("Encountered " + errors.size() + " errors, the first one is " + errors.get(0)); - } - } - @Test public void concurrentLoadFileMetadata() throws Exception { runLoadMetadata(null, false, true, false); @@ -192,11 +165,6 @@ public void concurrentLoadFileMetadataExistingDir() throws Exception { runLoadMetadata(WritePType.CACHE_THROUGH, false, true, false); } - @Test - public void concurrentLoadFileMetadataNonPersistedDir() throws Exception { - runLoadMetadata(WritePType.MUST_CACHE, false, true, false); - } - @Test public void concurrentLoadSameFileMetadata() throws Exception { runLoadMetadata(null, true, true, false); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/BufferedBlockInStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/BufferedBlockInStreamIntegrationTest.java index 3041168eab0e..5f63edda6c55 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/BufferedBlockInStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/BufferedBlockInStreamIntegrationTest.java @@ -69,8 +69,6 @@ private static List getOptionSet() { CreateFilePOptions optionsDefault = CreateFilePOptions.getDefaultInstance(); ret.add(optionsDefault.toBuilder().setWriteType(WritePType.CACHE_THROUGH) .setRecursive(true).build()); - ret.add(optionsDefault.toBuilder().setWriteType(WritePType.MUST_CACHE).setRecursive(true) - .build()); ret.add(optionsDefault.toBuilder().setWriteType(WritePType.THROUGH).setRecursive(true) .build()); return ret; diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamDecommissionIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamDecommissionIntegrationTest.java index ff7fd0f63aa5..0d3a32a05bc8 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamDecommissionIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamDecommissionIntegrationTest.java @@ -13,7 +13,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; import alluxio.AlluxioURI; import alluxio.annotation.dora.DoraTestTodoItem; @@ -25,7 +24,6 @@ import alluxio.client.file.URIStatus; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; -import alluxio.exception.status.UnavailableException; import alluxio.grpc.CreateFilePOptions; import alluxio.grpc.DecommissionWorkerPOptions; import alluxio.grpc.GetStatusPOptions; @@ -74,14 +72,12 @@ public class FileInStreamDecommissionIntegrationTest { .build(); private FileSystem mFileSystem = null; private CreateFilePOptions mWriteBoth; - private CreateFilePOptions mWriteAlluxio; private OpenFilePOptions mReadNoCache; private OpenFilePOptions mReadCachePromote; private String mTestPath; private ExecutorService mThreadPool; private String mCacheThroughFilePath; - private String mMustCacheFilePath; @Rule public ExpectedException mThrown = ExpectedException.none(); @@ -100,25 +96,16 @@ public final void setUp() throws Exception { .setWriteType(WritePType.CACHE_THROUGH) .setWorkerLocation(GrpcUtils.toProto(worker1)) .setRecursive(true).build(); - mWriteAlluxio = CreateFilePOptions.newBuilder() - .setBlockSizeBytes(BLOCK_SIZE) - .setWriteType(WritePType.MUST_CACHE) - .setWorkerLocation(GrpcUtils.toProto(worker1)) - .setRecursive(true).build(); mReadCachePromote = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mReadNoCache = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); mTestPath = PathUtils.uniqPath(); mCacheThroughFilePath = mTestPath + "/file_BOTH"; - mMustCacheFilePath = mTestPath + "/file_CACHE"; // Create files of varying size and write type to later read from AlluxioURI path0 = new AlluxioURI(mCacheThroughFilePath); FileSystemTestUtils.createByteFile(mFileSystem, path0, mWriteBoth, LENGTH); - AlluxioURI path1 = new AlluxioURI(mMustCacheFilePath); - FileSystemTestUtils.createByteFile(mFileSystem, path1, mWriteAlluxio, LENGTH); - mThreadPool = Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("decommission-worker-%d", true)); } @@ -132,7 +119,6 @@ public final void tearDown() throws Exception { private List getOptionSet() { List ret = new ArrayList<>(2); ret.add(mWriteBoth); - ret.add(mWriteAlluxio); return ret; } @@ -197,43 +183,6 @@ public void readUfsFromUndecommissionedWorker() throws Exception { assertNotEquals(workerToDecommission, block1Locs.get(0).getWorkerAddress()); } - @Test - /* - * If a stream is created after the worker is decommissioned, it cannot pick that worker. - * And if that worker holds the only cache and the block is not in UFS, - * the read will fail. - */ - public void cannotReadCacheFromDecommissionedWorker() throws Exception { - AlluxioURI uri = new AlluxioURI(mMustCacheFilePath); - FileSystemContext context = FileSystemContext - .create(new TestUserState("test", Configuration.global()).getSubject(), - Configuration.global()); - List availableWorkers = context.acquireBlockMasterClientResource() - .get().getWorkerInfoList(); - assertEquals(2, availableWorkers.size()); - - URIStatus status = context.acquireMasterClientResource().get() - .getStatus(uri, GetStatusPOptions.getDefaultInstance()); - List blockInfos = status.getFileBlockInfos(); - FileBlockInfo block0 = blockInfos.get(0); - BlockLocation loc0 = block0.getBlockInfo().getLocations().get(0); - WorkerNetAddress targetWorker = loc0.getWorkerAddress(); - - DecommissionWorkerPOptions decomOptions = DecommissionWorkerPOptions.newBuilder() - .setWorkerHostname(targetWorker.getHost()).setWorkerWebPort(targetWorker.getWebPort()) - .setCanRegisterAgain(true).build(); - context.acquireBlockMasterClientResource().get().decommissionWorker(decomOptions); - - // This stream is able to find the undecommissioned worker and use that to read from UFS - FileInStream is = mFileSystem.openFile(uri, mReadCachePromote); - // The worker has been decommissioned and the file only exists in that worker - // So the client cannot read - assertThrows(UnavailableException.class, () -> { - int value = is.read(); - }); - is.close(); - } - @Test /* * The target worker is decommissioned while the stream is reading. diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamIntegrationTest.java index c4877c951385..15fbfcffe54b 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileInStreamIntegrationTest.java @@ -82,7 +82,6 @@ public static Collection data() { private FileSystem mFileSystem; private CreateFilePOptions mWriteBoth; - private CreateFilePOptions mWriteAlluxio; private CreateFilePOptions mWriteUnderStore; private String mTestPath; @@ -117,9 +116,6 @@ public void before() throws Exception { mWriteBoth = CreateFilePOptions.newBuilder().setMode(Mode.createFullAccess().toProto()) .setBlockSizeBytes(BLOCK_SIZE).setWriteType(WritePType.CACHE_THROUGH) .setRecursive(true).build(); - mWriteAlluxio = CreateFilePOptions.newBuilder().setMode(Mode.createFullAccess().toProto()) - .setBlockSizeBytes(BLOCK_SIZE).setWriteType(WritePType.MUST_CACHE).setRecursive(true) - .build(); mWriteUnderStore = CreateFilePOptions.newBuilder().setMode(Mode.createFullAccess().toProto()) .setBlockSizeBytes(BLOCK_SIZE).setWriteType(WritePType.THROUGH).setRecursive(true) .build(); @@ -137,7 +133,6 @@ public void before() throws Exception { private List getOptionSet() { List ret = new ArrayList<>(3); ret.add(mWriteBoth); - ret.add(mWriteAlluxio); ret.add(mWriteUnderStore); return ret; } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamDecommissionIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamDecommissionIntegrationTest.java index 896a9abf6cc6..76401f27fa8e 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamDecommissionIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamDecommissionIntegrationTest.java @@ -52,7 +52,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -78,14 +77,12 @@ public class FileOutStreamDecommissionIntegrationTest { .build(); private FileSystem mFileSystem = null; private CreateFilePOptions mWriteBoth; - private CreateFilePOptions mWriteAlluxio; private OpenFilePOptions mReadNoCache; private OpenFilePOptions mReadCachePromote; private String mTestPath; private ExecutorService mThreadPool; private String mCacheThroughFilePath; - private String mMustCacheFilePath; @Rule public ExpectedException mThrown = ExpectedException.none(); @@ -104,17 +101,11 @@ public final void setUp() throws Exception { .setWriteType(WritePType.CACHE_THROUGH) .setWorkerLocation(GrpcUtils.toProto(worker1)) .setRecursive(true).build(); - mWriteAlluxio = CreateFilePOptions.newBuilder() - .setBlockSizeBytes(BLOCK_SIZE) - .setWriteType(WritePType.MUST_CACHE) - .setWorkerLocation(GrpcUtils.toProto(worker1)) - .setRecursive(true).build(); mReadCachePromote = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mReadNoCache = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); mTestPath = PathUtils.uniqPath(); mCacheThroughFilePath = mTestPath + "/file_BOTH"; - mMustCacheFilePath = mTestPath + "/file_CACHE"; mThreadPool = Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("decommission-worker-%d", true)); } @@ -128,7 +119,6 @@ public final void tearDown() throws Exception { private List getOptionSet() { List ret = new ArrayList<>(2); ret.add(mWriteBoth); - ret.add(mWriteAlluxio); return ret; } @@ -421,84 +411,4 @@ public void halfCacheThroughStreamDecommission() throws Exception { } assertEquals(readLength, LENGTH); } - - @Test - /* - * When there is an active stream writing to one worker, decommission that worker. - * Then we make the stream wait a bit and realize that worker is no longer available. - * The stream is not able to continue because the cache target is gone. - */ - public void halfStreamMustCacheDecommission() throws Exception { - AlluxioURI uri = new AlluxioURI(mMustCacheFilePath); - - FileSystemContext context = FileSystemContext - .create(new TestUserState("test", Configuration.global()).getSubject(), - Configuration.global()); - List clusterWorkers = context.acquireBlockMasterClientResource() - .get().getWorkerInfoList(); - assertEquals(2, clusterWorkers.size()); - - WorkerNetAddress workerToDecommission = mLocalAlluxioClusterResource.get().getWorkerAddress(); - CountDownLatch streamActive = new CountDownLatch(1); - CountDownLatch workerDecommissioned = new CountDownLatch(1); - mThreadPool.submit(() -> { - try { - streamActive.await(); - DecommissionWorkerPOptions decomOptions = DecommissionWorkerPOptions.newBuilder() - .setWorkerHostname(workerToDecommission.getHost()) - .setWorkerWebPort(workerToDecommission.getWebPort()) - .setCanRegisterAgain(true).build(); - context.acquireBlockMasterClientResource().get().decommissionWorker(decomOptions); - List updatedWorkers = context.acquireBlockMasterClientResource() - .get().getWorkerInfoList(); - assertEquals(1, updatedWorkers.size()); - workerDecommissioned.countDown(); - } catch (Exception e) { - e.printStackTrace(); - } - }); - - // This stream is able to find the undecommissioned worker and use that to write to UFS - FileOutStream os = mFileSystem.createFile(uri, mWriteAlluxio); - byte[] ret = new byte[1024]; - // This has created the block stream that reads from the target worker - int writeLength = 0; - - boolean released = false; - while (writeLength < LENGTH) { - // 2 blocks on the same worker, decommission at the end of one BlockStream - // so when the FileStream continues, create the 2nd block stream where there's only one - // usable worker that does not have the block - if (writeLength == BLOCK_SIZE && !released) { - streamActive.countDown(); - released = true; - - // Wait a bit for the decommission to take effect - // After the worker is decommissioned, the stream can successfully complete - workerDecommissioned.await(); - - // Wait a bit for the worker list to refresh in the FileSystemContext - SleepUtils.sleepMs(CLIENT_WORKER_LIST_REFRESH_INTERVAL); - // The client realizes the target worker is decommissioned - List usableWorkers = context.getCachedWorkers(); - assertEquals(1, usableWorkers.size()); - // Continue the stream, the stream is not able to continue because the worker is - // no longer available - assertThrows(IOException.class, () -> { - os.write(ret); - }); - break; - } - os.write(ret); - writeLength += ret.length; - } - os.close(); - - // The worker has been decommissioned so the block locations are all empty - // No cache readable for the client - URIStatus statusAfterRead = context.acquireMasterClientResource() - .get().getStatus(uri, GetStatusPOptions.getDefaultInstance()); - // The stream was not successful so the length is 0 - assertEquals(0, statusAfterRead.getLength()); - } } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamIntegrationTest.java index 7f0985374140..991554ffc8f9 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/FileOutStreamIntegrationTest.java @@ -55,9 +55,7 @@ public final class FileOutStreamIntegrationTest extends AbstractFileOutStreamInt @Parameters public static Object[] dataFileInStreamIntegrationTest() { return new Object[] { - WriteType.ASYNC_THROUGH, WriteType.CACHE_THROUGH, - WriteType.MUST_CACHE, WriteType.THROUGH, }; } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fs/io/LocalBlockInStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fs/io/LocalBlockInStreamIntegrationTest.java index c156d517cabe..207fcb20a3cd 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fs/io/LocalBlockInStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fs/io/LocalBlockInStreamIntegrationTest.java @@ -53,7 +53,6 @@ public final class LocalBlockInStreamIntegrationTest extends BaseIntegrationTest new LocalAlluxioClusterResource.Builder().build(); private static FileSystem sFileSystem = null; private static CreateFilePOptions sWriteBoth; - private static CreateFilePOptions sWriteAlluxio; private static OpenFilePOptions sReadNoCache; private static OpenFilePOptions sReadCachePromote; private static String sTestPath; @@ -66,8 +65,6 @@ public static final void beforeClass() throws Exception { sFileSystem = sLocalAlluxioClusterResource.get().getClient(); sWriteBoth = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH) .setRecursive(true).build(); - sWriteAlluxio = CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE) - .setRecursive(true).build(); sReadCachePromote = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); sReadNoCache = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); @@ -85,7 +82,6 @@ public static final void beforeClass() throws Exception { private static List getOptionSet() { List ret = new ArrayList<>(2); ret.add(sWriteBoth); - ret.add(sWriteAlluxio); return ret; } diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/AbstractFuseIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/AbstractFuseIntegrationTest.java index 463a8c04ec80..237e8bcfbb24 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/AbstractFuseIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/AbstractFuseIntegrationTest.java @@ -31,7 +31,6 @@ import alluxio.fuse.AlluxioFuseUtils; import alluxio.grpc.OpenFilePOptions; import alluxio.grpc.ReadPType; -import alluxio.grpc.WritePType; import alluxio.master.LocalAlluxioCluster; import alluxio.testutils.IntegrationTestUtils; import alluxio.testutils.LocalAlluxioClusterResource; @@ -174,7 +173,7 @@ public void chgrp() throws Exception { String userName = System.getProperty("user.name"); Optional groupName = AlluxioFuseUtils.getGroupName(userName); Assert.assertTrue(groupName.isPresent()); - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 10); ShellUtils.execCommand("chgrp", groupName.get(), mMountPoint + testFile); assertEquals(groupName.get(), mFileSystem.getStatus(new AlluxioURI(testFile)).getGroup()); } @@ -183,7 +182,7 @@ public void chgrp() throws Exception { @Ignore public void chmod() throws Exception { String testFile = "/chmodTestFile"; - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 10); ShellUtils.execCommand("chmod", "777", mMountPoint + testFile); assertEquals((short) 0777, mFileSystem.getStatus(new AlluxioURI(testFile)).getMode()); } @@ -192,7 +191,7 @@ public void chmod() throws Exception { @Ignore public void chown() throws Exception { String testFile = "/chownTestFile"; - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 10); String userName = System.getProperty("user.name"); Optional groupName = AlluxioFuseUtils.getGroupName(userName); @@ -282,7 +281,7 @@ public void ls() throws Exception { public void mkdirAndMv() throws Exception { String testFile = "/mvTestFile"; String testFolder = "/mkdirTestFolder"; - FileSystemTestUtils.createByteFile(mFileSystem, testFile, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(mFileSystem, testFile, 10); ShellUtils.execCommand("mkdir", mMountPoint + testFolder); ShellUtils.execCommand("mv", mMountPoint + testFile, mMountPoint + testFolder + testFile); assertFalse(mFileSystem.exists(new AlluxioURI(testFile))); @@ -330,7 +329,7 @@ public void continueWithRevokedAuth() throws Exception { @Test @Ignore public void touchAndLs() throws Exception { - FileSystemTestUtils.createByteFile(mFileSystem, "/lsTestFile", WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(mFileSystem, "/lsTestFile", 10); String touchTestFile = "/touchTestFile"; ShellUtils.execCommand("touch", mMountPoint + touchTestFile); diff --git a/dora/tests/integration/src/test/java/alluxio/client/fuse/file/AbstractFuseFileStreamIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/fuse/file/AbstractFuseFileStreamIntegrationTest.java index 3f1858502aed..3ba7e33d5889 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/fuse/file/AbstractFuseFileStreamIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/fuse/file/AbstractFuseFileStreamIntegrationTest.java @@ -25,7 +25,6 @@ import alluxio.grpc.CreateFilePOptions; import alluxio.grpc.OpenFilePOptions; import alluxio.grpc.ReadPType; -import alluxio.grpc.WritePType; import alluxio.testutils.BaseIntegrationTest; import alluxio.testutils.LocalAlluxioClusterResource; import alluxio.util.io.BufferUtils; @@ -74,7 +73,7 @@ public void before() throws Exception { */ protected void writeIncreasingByteArrayToFile(AlluxioURI filePath, int fileLen) throws Exception { try (FileOutStream os = mFileSystem.createFile(filePath, CreateFilePOptions.newBuilder() - .setWriteType(WritePType.MUST_CACHE).setRecursive(true).build())) { + .setRecursive(true).build())) { os.write(BufferUtils.getIncreasingByteArray(fileLen)); } } diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java index b041e4e495e8..16df2b409239 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/ListStatusTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertTrue; import alluxio.AlluxioURI; -import alluxio.client.WriteType; import alluxio.client.file.FileSystem; import alluxio.client.file.URIStatus; import alluxio.conf.PropertyKey; @@ -69,7 +68,6 @@ public class ListStatusTest extends RestApiTest { .setProperty(PropertyKey.S3_REST_AUTHENTICATION_ENABLED, // TODO(czhu) refactor this key name false) // default, disables AWS "Authorization" header signature validation .setProperty(PropertyKey.USER_FILE_BUFFER_BYTES, "1KB") - .setProperty(PropertyKey.PROXY_S3_WRITE_TYPE, WriteType.MUST_CACHE.name()) // skip UFS .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "0") .setProperty(PropertyKey.PROXY_S3_TAGGING_RESTRICTIONS_ENABLED, true) // default .setProperty(PropertyKey.PROXY_S3_BUCKET_NAMING_RESTRICTIONS_ENABLED, false) // default diff --git a/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java b/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java index f1703a8a45c8..0eaf9ea41531 100644 --- a/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java +++ b/dora/tests/integration/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java @@ -116,7 +116,7 @@ public final class S3ClientRestApiTest extends RestApiTest { .setProperty(PropertyKey.S3_REST_AUTHENTICATION_ENABLED, // TODO(czhu) refactor this key name false) // default, disables AWS "Authorization" header signature validation .setProperty(PropertyKey.USER_FILE_BUFFER_BYTES, "1KB") - .setProperty(PropertyKey.PROXY_S3_WRITE_TYPE, WriteType.MUST_CACHE.name()) // skip UFS + .setProperty(PropertyKey.PROXY_S3_WRITE_TYPE, WriteType.CACHE_THROUGH.name()) // skip UFS .setProperty(PropertyKey.PROXY_S3_COMPLETE_MULTIPART_UPLOAD_MIN_PART_SIZE, "0") .setProperty(PropertyKey.PROXY_S3_TAGGING_RESTRICTIONS_ENABLED, true) // default .setProperty(PropertyKey.PROXY_S3_BUCKET_NAMING_RESTRICTIONS_ENABLED, false) // default diff --git a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/BlockMasterJournalIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/BlockMasterJournalIntegrationTest.java index 1e6369308c50..d1bfb5bb334e 100644 --- a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/BlockMasterJournalIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/BlockMasterJournalIntegrationTest.java @@ -20,7 +20,6 @@ import alluxio.client.file.FileSystemTestUtils; import alluxio.client.file.URIStatus; import alluxio.exception.BlockInfoException; -import alluxio.grpc.WritePType; import alluxio.master.AlluxioMasterProcess; import alluxio.master.LocalAlluxioCluster; import alluxio.master.block.BlockMaster; @@ -57,7 +56,7 @@ public void journalBlockCreation() throws Exception { BlockMaster blockMaster = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(BlockMaster.class); AlluxioURI file = new AlluxioURI("/test"); - FileSystemTestUtils.createByteFile(fs, file, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(fs, file, 10); URIStatus status = fs.getStatus(file); Long blockId = status.getBlockIds().get(0); assertNotNull(blockMaster.getBlockInfo(blockId)); @@ -74,7 +73,7 @@ public void journalBlockDeletion() throws Exception { BlockMaster blockMaster = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(BlockMaster.class); AlluxioURI file = new AlluxioURI("/test"); - FileSystemTestUtils.createByteFile(fs, file, WritePType.MUST_CACHE, 10); + FileSystemTestUtils.createByteFile(fs, file, 10); URIStatus status = fs.getStatus(file); Long blockId = status.getBlockIds().get(0); assertNotNull(blockMaster.getBlockInfo(blockId)); diff --git a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalBackupIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalBackupIntegrationTest.java index d76f1a15df18..b0810c2898c0 100644 --- a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalBackupIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalBackupIntegrationTest.java @@ -683,14 +683,14 @@ private void backupRestoreTest(boolean testFailover) throws Exception { AlluxioURI dir1 = new AlluxioURI("/dir1"); fs.createDirectory(dir1, - CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build()); + CreateDirectoryPOptions.newBuilder().build()); AlluxioURI backup1 = metaClient .backup(BackupPRequest.newBuilder().setTargetDirectory(backups.getAbsolutePath()) .setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()) .getBackupUri(); AlluxioURI dir2 = new AlluxioURI("/dir2"); fs.createDirectory(dir2, - CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build()); + CreateDirectoryPOptions.newBuilder().build()); AlluxioURI backup2 = metaClient .backup(BackupPRequest.newBuilder().setTargetDirectory(backups.getAbsolutePath()) .setOptions(BackupPOptions.newBuilder().setLocalFileSystem(false)).build()) @@ -746,7 +746,7 @@ private void backupDelegationTest(MultiProcessCluster.Builder clusterBuilder) th AlluxioURI dir1 = new AlluxioURI("/dir1"); mCluster.getFileSystemClient().createDirectory(dir1, - CreateDirectoryPOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build()); + CreateDirectoryPOptions.newBuilder().build()); AlluxioURI backupUri = waitForBackup(BackupPRequest.newBuilder() .setOptions(BackupPOptions.newBuilder().setLocalFileSystem(true)).build()); diff --git a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalIntegrationTest.java index 5db7d886ef36..c4802c9ce20e 100644 --- a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalIntegrationTest.java @@ -56,7 +56,7 @@ public class JournalIntegrationTest { new LocalAlluxioClusterResource.Builder() .setProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.UFS) .setProperty(PropertyKey.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS, 0) - .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE) + .setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH) .setNumWorkers(0) .build(); diff --git a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalShutdownIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalShutdownIntegrationTest.java index eae099c8dbfd..3edc56d8d624 100644 --- a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalShutdownIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/JournalShutdownIntegrationTest.java @@ -253,7 +253,7 @@ private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { LocalAlluxioCluster cluster = new LocalAlluxioCluster(); cluster.initConfiguration( IntegrationTestUtils.getTestName(getClass().getSimpleName(), mTestName.getMethodName())); - Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); + Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH); cluster.start(); return cluster; } diff --git a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/ufs/UfsJournalIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/ufs/UfsJournalIntegrationTest.java index bd20e89372cc..1c73164660f9 100644 --- a/dora/tests/integration/src/test/java/alluxio/server/ft/journal/ufs/UfsJournalIntegrationTest.java +++ b/dora/tests/integration/src/test/java/alluxio/server/ft/journal/ufs/UfsJournalIntegrationTest.java @@ -437,7 +437,7 @@ public void persistDirectoryLater() throws Exception { }; CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true) - .setWriteType(WritePType.MUST_CACHE).build(); + .build(); for (String directory : directories) { mFileSystem.createDirectory(new AlluxioURI(directory), options); }