From ed5a652d8b338e380b87966bdf729bbef28d0eec Mon Sep 17 00:00:00 2001 From: Joe Abraham Date: Thu, 1 Feb 2024 10:48:11 +0530 Subject: [PATCH] Introduce utility class for encoding --- velox/common/encode/Base64.cpp | 22 +--- velox/common/encode/EncoderUtils.h | 117 ++++++++++++++++++ velox/common/encode/tests/Base64Test.cpp | 53 -------- velox/common/encode/tests/CMakeLists.txt | 2 +- .../common/encode/tests/EncoderUtilsTests.cpp | 76 ++++++++++++ 5 files changed, 198 insertions(+), 72 deletions(-) create mode 100644 velox/common/encode/EncoderUtils.h create mode 100644 velox/common/encode/tests/EncoderUtilsTests.cpp diff --git a/velox/common/encode/Base64.cpp b/velox/common/encode/Base64.cpp index 12191cb768330..f17647f639ea3 100644 --- a/velox/common/encode/Base64.cpp +++ b/velox/common/encode/Base64.cpp @@ -90,7 +90,7 @@ constexpr const ReverseIndex kBase64UrlReverseIndexTable = { // Verify that for every entry in kBase64Charset, the corresponding entry // in kBase64ReverseIndexTable is correct. static_assert( - Base64::checkForwardIndex( + checkForwardIndex( sizeof(kBase64Charset) - 1, kBase64Charset, kBase64ReverseIndexTable), @@ -99,30 +99,16 @@ static_assert( // Verify that for every entry in kBase64UrlCharset, the corresponding entry // in kBase64UrlReverseIndexTable is correct. static_assert( - Base64::checkForwardIndex( + checkForwardIndex( sizeof(kBase64UrlCharset) - 1, kBase64UrlCharset, kBase64UrlReverseIndexTable), "kBase64UrlCharset has incorrect entries"); -// static -const bool Base64::findCharacterInCharSet( - const Charset& charset, - int base, - uint8_t idx, - const char c) { - for (; idx < base; ++idx) { - if (charset[idx] == c) { - return true; - } - } - return false; -} - // Verify that for every entry in kBase64ReverseIndexTable, the corresponding // entry in kBase64Charset is correct. static_assert( - Base64::checkReverseIndex( + checkReverseIndex( sizeof(kBase64ReverseIndexTable) - 1, kBase64Charset, kBase, @@ -132,7 +118,7 @@ static_assert( // Verify that for every entry in kBase64ReverseIndexTable, the corresponding // entry in kBase64Charset is correct. static_assert( - Base64::checkReverseIndex( + checkReverseIndex( sizeof(kBase64UrlReverseIndexTable) - 1, kBase64UrlCharset, kBase, diff --git a/velox/common/encode/EncoderUtils.h b/velox/common/encode/EncoderUtils.h new file mode 100644 index 0000000000000..683eb491063d8 --- /dev/null +++ b/velox/common/encode/EncoderUtils.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include + +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::encoding { + +const size_t kCharsetSize = 64; +const size_t kReverseIndexSize = 256; + +/// Character set used for encoding purposes. +/// Contains specific characters that form the encoding scheme. +using Charset = std::array; + +/// Reverse lookup table for decoding purposes. +/// Maps each possible encoded character to its corresponding numeric value +/// within the encoding base. +using ReverseIndex = std::array; + +/// Padding character used in encoding. +const static char kPadding = '='; +/// Checks if there is padding in encoded data. +static inline bool isPadded(const char* data, size_t len) { + return (len > 0 && data[len - 1] == kPadding) ? true : false; +} + +/// Counts the number of padding characters in encoded data. +static inline size_t numPadding(const char* src, size_t len) { + size_t numPadding{0}; + while (len > 0 && src[len - 1] == kPadding) { + numPadding++; + len--; + } + return numPadding; +} +/// Performs a reverse lookup in the reverse index to retrieve the original +/// index of a character in the base. +inline uint8_t +baseReverseLookup(int base, char p, const ReverseIndex& reverseIndex) { + auto curr = reverseIndex[(uint8_t)p]; + if (curr >= base) { + VELOX_USER_FAIL("decode() - invalid input string: invalid characters"); + } + return curr; +} + +// Validate the character in charset with ReverseIndex table +static constexpr bool checkForwardIndex( + uint8_t idx, + const Charset& charset, + const ReverseIndex& reverseIndex) { + for (uint8_t i = 0; i <= idx; ++i) { + if (!(reverseIndex[static_cast(charset[i])] == i)) { + return false; + } + } + return true; +} + +/// Searches for a character within a charset up to a certain index. +static const bool findCharacterInCharSet( + const Charset& charset, + int base, + uint8_t idx, + const char c) { + for (; idx < base; ++idx) { + if (charset[idx] == c) { + return true; + } + } + return false; +} + +/// Checks the consistency of a reverse index mapping for a given character +/// set. +static constexpr bool checkReverseIndex( + uint8_t idx, + const Charset& charset, + int base, + const ReverseIndex& reverseIndex) { + for (uint8_t currentIdx = idx; currentIdx != static_cast(-1); + --currentIdx) { + if (reverseIndex[currentIdx] == 255) { + if (findCharacterInCharSet( + charset, base, 0, static_cast(currentIdx))) { + return false; + } + } else { + if (!(charset[reverseIndex[currentIdx]] == currentIdx)) { + return false; + } + } + } + return true; +} + +} // namespace facebook::velox::encoding diff --git a/velox/common/encode/tests/Base64Test.cpp b/velox/common/encode/tests/Base64Test.cpp index e31ddef34b1b9..ebd87168d8117 100644 --- a/velox/common/encode/tests/Base64Test.cpp +++ b/velox/common/encode/tests/Base64Test.cpp @@ -86,57 +86,4 @@ TEST_F(Base64Test, calculateDecodedSizeProperSize) { EXPECT_EQ(14, encoded_size); } -TEST_F(Base64Test, ChecksPadding) { - EXPECT_TRUE(Base64::isPadded("ABC=", 4)); - EXPECT_FALSE(Base64::isPadded("ABC", 3)); -} - -TEST_F(Base64Test, CountsPaddingCorrectly) { - EXPECT_EQ(0, Base64::numPadding("ABC", 3)); - EXPECT_EQ(1, Base64::numPadding("ABC=", 4)); - EXPECT_EQ(2, Base64::numPadding("AB==", 4)); -} - -constexpr Charset testCharset = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - -constexpr ReverseIndex testReverseIndex = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, - 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -TEST_F(Base64Test, HandlesLookupAndExceptions) { - int base = 64; - EXPECT_NO_THROW(baseReverseLookup(base, 'A', testReverseIndex)); - EXPECT_THROW(baseReverseLookup(base, '=', testReverseIndex), VeloxUserError); -} - -TEST_F(Base64Test, ValidatesCharsetWithReverseIndex) { - EXPECT_TRUE(Base64::checkForwardIndex(63, testCharset, testReverseIndex)); -} - -TEST_F(Base64Test, ValidatesReverseIndexWithCharset) { - EXPECT_TRUE( - Base64::checkReverseIndex(255, testCharset, 64, testReverseIndex)); -} - } // namespace facebook::velox::encoding diff --git a/velox/common/encode/tests/CMakeLists.txt b/velox/common/encode/tests/CMakeLists.txt index e3268cb7f1b96..548ce580e4691 100644 --- a/velox/common/encode/tests/CMakeLists.txt +++ b/velox/common/encode/tests/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(velox_common_encode_test Base64Test.cpp) +add_executable(velox_common_encode_test Base64Test.cpp EncoderUtilsTests.cpp) add_test(velox_common_encode_test velox_common_encode_test) target_link_libraries( velox_common_encode_test diff --git a/velox/common/encode/tests/EncoderUtilsTests.cpp b/velox/common/encode/tests/EncoderUtilsTests.cpp new file mode 100644 index 0000000000000..70b526d332fbf --- /dev/null +++ b/velox/common/encode/tests/EncoderUtilsTests.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "velox/common/base/tests/GTestUtils.h" +#include "velox/common/encode/EncoderUtils.h" + +namespace facebook::velox::encoding { +class EncoderUtilsTest : public ::testing::Test {}; + +TEST_F(EncoderUtilsTest, isPadded) { + EXPECT_TRUE(isPadded("ABC=", 4)); + EXPECT_FALSE(isPadded("ABC", 3)); +} + +TEST_F(EncoderUtilsTest, numPadding) { + EXPECT_EQ(0, numPadding("ABC", 3)); + EXPECT_EQ(1, numPadding("ABC=", 4)); + EXPECT_EQ(2, numPadding("AB==", 4)); +} + +constexpr Charset testCharset = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +constexpr ReverseIndex testReverseIndex = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +TEST_F(EncoderUtilsTest, baseReverseLookup) { + int base = 64; + EXPECT_NO_THROW(baseReverseLookup(base, 'A', testReverseIndex)); + EXPECT_THROW(baseReverseLookup(base, '=', testReverseIndex), std::exception); +} + +TEST_F(EncoderUtilsTest, checkForwardIndex) { + EXPECT_TRUE(checkForwardIndex(63, testCharset, testReverseIndex)); +} + +TEST_F(EncoderUtilsTest, checkReverseIndex) { + EXPECT_TRUE(checkReverseIndex(255, testCharset, 64, testReverseIndex)); +} + +} // namespace facebook::velox::encoding