diff --git a/velox/common/encode/EncoderUtils.h b/velox/common/encode/EncoderUtils.h new file mode 100644 index 0000000000000..b69f6f4cf99eb --- /dev/null +++ b/velox/common/encode/EncoderUtils.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include + +namespace facebook::velox::encoding { + +class EncoderException : public std::exception { + public: + explicit EncoderException(const char* msg) : msg_(msg) {} + const char* what() const noexcept override { + return msg_; + } + + protected: + const char* msg_; +}; + +using Charset = std::array; +using ReverseIndex = std::array; + +// Padding character used in encoding +constexpr static char kPadding = '='; + +// Checks is there padding in encoded data +inline bool isPadded(const char* data, size_t len) { + return (len > 0 && data[len - 1] == kPadding) ? true : false; +} + +// Counts the number of padding characters in encoded data. +inline size_t countPadding(const char* src, size_t len) { + size_t padding_count = 0; + while (len > 0 && src[len - 1] == kPadding) { + padding_count++; + len--; + } + + return padding_count; +} + +// Gets value corresponding to an encoded character +inline uint8_t +baseReverseLookup(int base, char p, const ReverseIndex& reverse_lookup) { + auto curr = reverse_lookup[(uint8_t)p]; + // Value of encoded character shall be less than base. + if (curr >= base) { + throw EncoderException( + "decode() - invalid input string: invalid characters"); + } + + return curr; +} + +// Validate the character in charset with ReverseIndex table +constexpr bool checkForwardIndex( + uint8_t idx, + const Charset& charset, + const ReverseIndex& table) { + return (table[static_cast(charset[idx])] == idx) && + (idx > 0 ? checkForwardIndex(idx - 1, charset, table) : true); +} + +/// Similar to strchr(), but for null-terminated const strings. +/// Another difference is that we do not consider "\0" to be present in the +/// string. +/// Returns true if "str" contains the character c. +constexpr bool findCharacterInCharSet( + const Charset& charset, + int base, + uint8_t idx, + const char c) { + return idx < base && + ((charset[idx] == c) || + findCharacterInCharSet(charset, base, idx + 1, c)); +} + +// Validate the value in ReverseIndex table with charset. +constexpr bool checkReverseIndex( + uint8_t idx, + const Charset& charset, + int base, + const ReverseIndex& table) { + return (table[idx] == 255 ? !findCharacterInCharSet( + charset, base, 0, static_cast(idx)) + : (charset[table[idx]] == idx)) && + (idx > 0 ? checkReverseIndex(idx - 1, charset, base, table) : true); +} + +} // namespace facebook::velox::encoding diff --git a/velox/functions/prestosql/tests/BinaryFunctionsTest.cpp b/velox/functions/prestosql/tests/BinaryFunctionsTest.cpp index 0a647d2ecf520..e64cee65dd2d4 100644 --- a/velox/functions/prestosql/tests/BinaryFunctionsTest.cpp +++ b/velox/functions/prestosql/tests/BinaryFunctionsTest.cpp @@ -18,6 +18,7 @@ #include #include #include "velox/common/base/VeloxException.h" +#include "velox/common/encode/EncoderUtils.h" #include "velox/expression/Expr.h" #include "velox/functions/prestosql/tests/utils/FunctionBaseTest.h" @@ -40,6 +41,7 @@ std::string hexToDec(const std::string& str) { } class BinaryFunctionsTest : public FunctionBaseTest {}; +class EncoderUtilsTest : public FunctionBaseTest {}; TEST_F(BinaryFunctionsTest, md5) { const auto md5 = [&](std::optional arg) { @@ -697,4 +699,57 @@ TEST_F(BinaryFunctionsTest, toIEEE754Bits32) { hexToDec("FF7FFFFF"), toIEEE754Bits32(std::numeric_limits::lowest())); } + +TEST_F(BinaryFunctionsTest, ChecksPadding) { + EXPECT_TRUE(facebook::velox::encoding::isPadded("ABC=", 4)); + EXPECT_FALSE(facebook::velox::encoding::isPadded("ABC", 3)); +} + +TEST_F(BinaryFunctionsTest, CountsPaddingCorrectly) { + EXPECT_EQ(0, facebook::velox::encoding::countPadding("ABC", 3)); + EXPECT_EQ(1, facebook::velox::encoding::countPadding("ABC=", 4)); + EXPECT_EQ(2, facebook::velox::encoding::countPadding("AB==", 4)); +} + +constexpr facebook::velox::encoding::Charset testCharset = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +constexpr facebook::velox::encoding::ReverseIndex testReverseIndex = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +TEST_F(BinaryFunctionsTest, HandlesLookupAndExceptions) { + int base = 64; + EXPECT_NO_THROW(facebook::velox::encoding::baseReverseLookup(base, 'A', testReverseIndex)); + EXPECT_THROW(facebook::velox::encoding::baseReverseLookup(base, '=', testReverseIndex), facebook::velox::encoding::EncoderException); +} + +TEST_F(BinaryFunctionsTest, ValidatesCharsetWithReverseIndex) { + EXPECT_TRUE(facebook::velox::encoding::checkForwardIndex(63, testCharset, testReverseIndex)); +} + +TEST_F(BinaryFunctionsTest, ValidatesReverseIndexWithCharset) { + EXPECT_TRUE(facebook::velox::encoding::checkReverseIndex(255, testCharset, 64, testReverseIndex)); +} + } // namespace