From e394d740769730f6960e7022e0bc8f4dee1e2e9b Mon Sep 17 00:00:00 2001 From: Zkplo <87751516+Zkplo@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:39:27 +0800 Subject: [PATCH] [INLONG-10956][SDK] Transform SQL supports SHA encryption algorithm (#10996) Co-authored-by: ZKpLo <14148880+zkplo@user.noreply.gitee.com> --- .../process/function/Sha2Function.java | 75 ++++++++++++++++++ .../process/function/ShaFunction.java | 54 +++++++++++++ ...TransformArithmeticFunctionsProcessor.java | 78 +++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java new file mode 100644 index 00000000000..ad120712bfd --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import org.apache.commons.codec.digest.DigestUtils; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.apache.commons.codec.digest.MessageDigestAlgorithms.SHA_224; + +/** + * Sha2Function + * description: SHA2(str, hash_length): Calculates the SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512) + * return NULL If either argument is NULL or the hash length(224 256 384 512) is not one of the permitted values + * return a hash value containing the desired number of bits. + */ +@TransformFunction(names = {"sha2"}) +public class Sha2Function implements ValueParser { + + private final ValueParser msgParser; + private final ValueParser lenParser; + + public Sha2Function(Function expr) { + List expressions = expr.getParameters().getExpressions(); + msgParser = OperatorTools.buildParser(expressions.get(0)); + lenParser = OperatorTools.buildParser(expressions.get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object msgObj = msgParser.parse(sourceData, rowIndex, context); + Object lenObj = lenParser.parse(sourceData, rowIndex, context); + if (msgObj == null || lenObj == null) { + return null; + } + String msg = msgObj.toString(); + int len = Integer.parseInt(lenObj.toString()); + switch (len) { + case 0: + case 256: + return DigestUtils.sha256Hex(msg.getBytes(StandardCharsets.UTF_8)); + case 224: + return new DigestUtils(SHA_224).digestAsHex(msg.getBytes(StandardCharsets.UTF_8)); + case 384: + return DigestUtils.sha384Hex(msg.getBytes(StandardCharsets.UTF_8)); + case 512: + return DigestUtils.sha512Hex(msg.getBytes(StandardCharsets.UTF_8)); + default: + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java new file mode 100644 index 00000000000..7da5e6c3a9d --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; +import org.apache.commons.codec.digest.DigestUtils; + +import java.nio.charset.StandardCharsets; + +/** + * ShaFunction + * description: sha(string): Compute the SHA-1 160 bit checksum of a string. + * return NULL if the parameter is NULL + * return a string of 40 hexadecimal digits. + */ +@TransformFunction(names = {"sha"}) +public class ShaFunction implements ValueParser { + + private final ValueParser msgParser; + + public ShaFunction(Function expr) { + msgParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object msgObj = msgParser.parse(sourceData, rowIndex, context); + if (msgObj == null) { + return null; + } + String msg = msgObj.toString(); + return DigestUtils.sha1Hex(msg.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index dcdb2d61d45..738ef2e0f6d 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -287,6 +287,84 @@ public void testIfNullFunction() throws Exception { Assert.assertEquals("result=null", output.get(0)); } + @Test + public void testShaFunction() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: sha("") + transformSql = "select sha(numeric1) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=da39a3ee5e6b4b0d3255bfef95601890afd80709", output.get(0)); + + // case2: sha("5") + data = "5|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=ac3478d69a3c81fa62e60f5c3696165a4e5e6ac4", output.get(0)); + + // case3: sha(null) + transformSql = "select sha(xxd) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "3|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + + @Test + public void testSha2Function() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: sha2("",3) + transformSql = "select sha2(numeric1,numeric2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case2: sha2("5",224) + data = "5|224|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=b51d18b551043c1f145f22dbde6f8531faeaf68c54ed9dd79ce24d17", output.get(0)); + + // case3: sha2("5",0) + data = "5|0|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", output.get(0)); + + // case4: sha2(null,224) + transformSql = "select sha2(xxd,224) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "3|224|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + @Test public void testMd5Function() throws Exception { String transformSql = "select md5(numeric1) from source";