diff --git a/CHANGELOG.md b/CHANGELOG.md index acef24204..bb053d1de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file. - hadoop: Build from source ([#526]). - superset: Add patch that fixes saved queries export ([#539]). - inotify-tools: Download from Nexus instead of using the EPEL 8 repository ([#549]). +- hadoop: Add patches to fix missing operationType for some operations in authorizer ([#555]). - airflow: bump git-sync to `4.2.1` ([#562]). ### Removed @@ -64,6 +65,7 @@ All notable changes to this project will be documented in this file. [#526]: https://github.com/stackabletech/docker-images/pull/526 [#529]: https://github.com/stackabletech/docker-images/pull/529 [#531]: https://github.com/stackabletech/docker-images/pull/531 +[#533]: https://github.com/stackabletech/docker-images/pull/533 [#534]: https://github.com/stackabletech/docker-images/pull/534 [#536]: https://github.com/stackabletech/docker-images/pull/536 [#537]: https://github.com/stackabletech/docker-images/pull/537 @@ -75,7 +77,7 @@ All notable changes to this project will be documented in this file. [#547]: https://github.com/stackabletech/docker-images/pull/547 [#549]: https://github.com/stackabletech/docker-images/pull/549 [#551]: https://github.com/stackabletech/docker-images/pull/551 -[#533]: https://github.com/stackabletech/docker-images/pull/533 +[#555]: https://github.com/stackabletech/docker-images/pull/555 [#558]: https://github.com/stackabletech/docker-images/pull/558 [#559]: https://github.com/stackabletech/docker-images/pull/559 [#560]: https://github.com/stackabletech/docker-images/pull/560 diff --git a/hadoop/stackable/patches/3.3.4/006-HDFS-17378-3.3.4.patch b/hadoop/stackable/patches/3.3.4/006-HDFS-17378-3.3.4.patch new file mode 100644 index 000000000..5d43cfb93 --- /dev/null +++ b/hadoop/stackable/patches/3.3.4/006-HDFS-17378-3.3.4.patch @@ -0,0 +1,191 @@ +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +index 9855b434e9c4..b3781ee1dd26 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +@@ -2530,15 +2530,16 @@ void unsetStoragePolicy(String src) throws IOException { + * @throws IOException + */ + BlockStoragePolicy getStoragePolicy(String src) throws IOException { ++ final String operationName = "getStoragePolicy"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getStoragePolicy(dir, pc, blockManager, src); + } finally { +- readUnlock("getStoragePolicy"); ++ readUnlock(operationName); + } + } + +@@ -2558,15 +2559,16 @@ BlockStoragePolicy[] getStoragePolicies() throws IOException { + } + + long getPreferredBlockSize(String src) throws IOException { ++ final String operationName = "getPreferredBlockSize"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getPreferredBlockSize(dir, pc, src); + } finally { +- readUnlock("getPreferredBlockSize"); ++ readUnlock(operationName); + } + } + +@@ -2619,7 +2621,6 @@ HdfsFileStatus startFile(String src, PermissionStatus permissions, + boolean createParent, short replication, long blockSize, + CryptoProtocolVersion[] supportedVersions, String ecPolicyName, + String storagePolicy, boolean logRetryCache) throws IOException { +- + HdfsFileStatus status; + try { + status = startFileInt(src, permissions, holder, clientMachine, flag, +@@ -2639,6 +2640,7 @@ private HdfsFileStatus startFileInt(String src, + long blockSize, CryptoProtocolVersion[] supportedVersions, + String ecPolicyName, String storagePolicy, boolean logRetryCache) + throws IOException { ++ final String operationName = "create"; + if (NameNode.stateChangeLog.isDebugEnabled()) { + StringBuilder builder = new StringBuilder(); + builder.append("DIR* NameSystem.startFile: src=").append(src) +@@ -2676,7 +2678,7 @@ private HdfsFileStatus startFileInt(String src, + + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2740,7 +2742,7 @@ private HdfsFileStatus startFileInt(String src, + dir.writeUnlock(); + } + } finally { +- writeUnlock("create"); ++ writeUnlock(operationName); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -2769,10 +2771,11 @@ private HdfsFileStatus startFileInt(String src, + */ + boolean recoverLease(String src, String holder, String clientMachine) + throws IOException { ++ final String operationName = "recoverLease"; + boolean skipSync = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2793,7 +2796,7 @@ boolean recoverLease(String src, String holder, String clientMachine) + skipSync = true; + throw se; + } finally { +- writeUnlock("recoverLease"); ++ writeUnlock(operationName); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -3010,6 +3013,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + final Set excludes, + final int numAdditionalNodes, final String clientName + ) throws IOException { ++ final String operationName = "getAdditionalDatanode"; + //check if the feature is enabled + dtpReplaceDatanodeOnFailure.checkEnabled(); + +@@ -3021,7 +3025,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + final BlockType blockType; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + // Changing this operation category to WRITE instead of making getAdditionalDatanode as a +@@ -3047,7 +3051,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + "src=%s, fileId=%d, blk=%s, clientName=%s, clientMachine=%s", + src, fileId, blk, clientName, clientMachine)); + } finally { +- readUnlock("getAdditionalDatanode"); ++ readUnlock(operationName); + } + + if (clientnode == null) { +@@ -3069,11 +3073,12 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + */ + void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) + throws IOException { ++ final String operationName = "abandonBlock"; + NameNode.stateChangeLog.debug( + "BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3082,7 +3087,7 @@ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) + NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} is " + + "removed from pendingCreates", b); + } finally { +- writeUnlock("abandonBlock"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + } +@@ -3136,10 +3141,11 @@ INodeFile checkLease(INodesInPath iip, String holder, long fileId) + boolean completeFile(final String src, String holder, + ExtendedBlock last, long fileId) + throws IOException { ++ final String operationName = "completeFile"; + boolean success = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3147,7 +3153,7 @@ boolean completeFile(final String src, String holder, + success = FSDirWriteFileOp.completeFile(this, pc, src, holder, last, + fileId); + } finally { +- writeUnlock("completeFile"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + if (success) { +@@ -3572,10 +3578,11 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type) + */ + void fsync(String src, long fileId, String clientName, long lastBlockLength) + throws IOException { ++ final String operationName = "fsync"; + NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3589,7 +3596,7 @@ void fsync(String src, long fileId, String clientName, long lastBlockLength) + } + FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false); + } finally { +- writeUnlock("fsync"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + } diff --git a/hadoop/stackable/patches/3.3.6/006-HDFS-17378-3.3.6.patch b/hadoop/stackable/patches/3.3.6/006-HDFS-17378-3.3.6.patch new file mode 100644 index 000000000..5d43cfb93 --- /dev/null +++ b/hadoop/stackable/patches/3.3.6/006-HDFS-17378-3.3.6.patch @@ -0,0 +1,191 @@ +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +index 9855b434e9c4..b3781ee1dd26 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +@@ -2530,15 +2530,16 @@ void unsetStoragePolicy(String src) throws IOException { + * @throws IOException + */ + BlockStoragePolicy getStoragePolicy(String src) throws IOException { ++ final String operationName = "getStoragePolicy"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getStoragePolicy(dir, pc, blockManager, src); + } finally { +- readUnlock("getStoragePolicy"); ++ readUnlock(operationName); + } + } + +@@ -2558,15 +2559,16 @@ BlockStoragePolicy[] getStoragePolicies() throws IOException { + } + + long getPreferredBlockSize(String src) throws IOException { ++ final String operationName = "getPreferredBlockSize"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getPreferredBlockSize(dir, pc, src); + } finally { +- readUnlock("getPreferredBlockSize"); ++ readUnlock(operationName); + } + } + +@@ -2619,7 +2621,6 @@ HdfsFileStatus startFile(String src, PermissionStatus permissions, + boolean createParent, short replication, long blockSize, + CryptoProtocolVersion[] supportedVersions, String ecPolicyName, + String storagePolicy, boolean logRetryCache) throws IOException { +- + HdfsFileStatus status; + try { + status = startFileInt(src, permissions, holder, clientMachine, flag, +@@ -2639,6 +2640,7 @@ private HdfsFileStatus startFileInt(String src, + long blockSize, CryptoProtocolVersion[] supportedVersions, + String ecPolicyName, String storagePolicy, boolean logRetryCache) + throws IOException { ++ final String operationName = "create"; + if (NameNode.stateChangeLog.isDebugEnabled()) { + StringBuilder builder = new StringBuilder(); + builder.append("DIR* NameSystem.startFile: src=").append(src) +@@ -2676,7 +2678,7 @@ private HdfsFileStatus startFileInt(String src, + + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2740,7 +2742,7 @@ private HdfsFileStatus startFileInt(String src, + dir.writeUnlock(); + } + } finally { +- writeUnlock("create"); ++ writeUnlock(operationName); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -2769,10 +2771,11 @@ private HdfsFileStatus startFileInt(String src, + */ + boolean recoverLease(String src, String holder, String clientMachine) + throws IOException { ++ final String operationName = "recoverLease"; + boolean skipSync = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2793,7 +2796,7 @@ boolean recoverLease(String src, String holder, String clientMachine) + skipSync = true; + throw se; + } finally { +- writeUnlock("recoverLease"); ++ writeUnlock(operationName); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -3010,6 +3013,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + final Set excludes, + final int numAdditionalNodes, final String clientName + ) throws IOException { ++ final String operationName = "getAdditionalDatanode"; + //check if the feature is enabled + dtpReplaceDatanodeOnFailure.checkEnabled(); + +@@ -3021,7 +3025,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + final BlockType blockType; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + // Changing this operation category to WRITE instead of making getAdditionalDatanode as a +@@ -3047,7 +3051,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + "src=%s, fileId=%d, blk=%s, clientName=%s, clientMachine=%s", + src, fileId, blk, clientName, clientMachine)); + } finally { +- readUnlock("getAdditionalDatanode"); ++ readUnlock(operationName); + } + + if (clientnode == null) { +@@ -3069,11 +3073,12 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, + */ + void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) + throws IOException { ++ final String operationName = "abandonBlock"; + NameNode.stateChangeLog.debug( + "BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3082,7 +3087,7 @@ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) + NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} is " + + "removed from pendingCreates", b); + } finally { +- writeUnlock("abandonBlock"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + } +@@ -3136,10 +3141,11 @@ INodeFile checkLease(INodesInPath iip, String holder, long fileId) + boolean completeFile(final String src, String holder, + ExtendedBlock last, long fileId) + throws IOException { ++ final String operationName = "completeFile"; + boolean success = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3147,7 +3153,7 @@ boolean completeFile(final String src, String holder, + success = FSDirWriteFileOp.completeFile(this, pc, src, holder, last, + fileId); + } finally { +- writeUnlock("completeFile"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + if (success) { +@@ -3572,10 +3578,11 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type) + */ + void fsync(String src, long fileId, String clientName, long lastBlockLength) + throws IOException { ++ final String operationName = "fsync"; + NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3589,7 +3596,7 @@ void fsync(String src, long fileId, String clientName, long lastBlockLength) + } + FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false); + } finally { +- writeUnlock("fsync"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + }