refactor: rename tokenize to lex

midouest · Feb 28, 2021 · 9798f75 · 9798f75
1 parent fca5654
commit 9798f75
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 22 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -55,7 +55,7 @@ option(TEST "Build Catch2 test cases" ON)
 
 set(common_cpp_files
     src/parse.cpp
-    src/tokenize.cpp
+    src/lex.cpp
 )
 
 # bytebeat target
@@ -72,8 +72,8 @@ endif()
 if(TEST)
     set(test_cpp_files
         test/test_expression.cpp
+        test/test_lex.cpp
         test/test_parse.cpp
-        test/test_tokenize.cpp
     )
     find_package(Catch2 REQUIRED)
     add_executable(

diff --git a/include/tokenize.hpp → include/lex.hpp b/include/tokenize.hpp → include/lex.hpp
@@ -7,5 +7,5 @@ using namespace std;
 
 namespace bb
 {
-    vector<string> tokenize(string &input);
+    vector<string> lex(string &input);
 }
diff --git a/src/tokenize.cpp → src/lex.cpp b/src/tokenize.cpp → src/lex.cpp
@@ -2,7 +2,7 @@
 #include <vector>
 #include <stdexcept>
 
-#include "tokenize.hpp"
+#include "lex.hpp"
 
 using namespace std;
 
@@ -11,7 +11,7 @@ namespace bb
     bool is_whitespace(char c);
     bool is_terminal(char c);
 
-    vector<string> tokenize(string &input)
+    vector<string> lex(string &input)
     {
         vector<string> tokens;
 

diff --git a/src/parse.cpp b/src/parse.cpp
@@ -1,5 +1,5 @@
 #include "parse.hpp"
-#include "tokenize.hpp"
+#include "lex.hpp"
 
 #include <iostream>
 
@@ -19,7 +19,7 @@ namespace bb
 
     ExpressionPtr parse(string &input)
     {
-        vector<string> tokens = tokenize(input);
+        vector<string> tokens = lex(input);
         if (tokens.empty())
         {
             throw invalid_argument("no tokens to parse");

diff --git a/test/test_tokenize.cpp → test/test_lex.cpp b/test/test_tokenize.cpp → test/test_lex.cpp
@@ -1,51 +1,51 @@
 #include <catch2/catch.hpp>
-#include "tokenize.hpp"
+#include "lex.hpp"
 
 using namespace std;
 using namespace bb;
 
-TEST_CASE("tokenize empty input", "[tokenize]")
+TEST_CASE("lex empty input", "[lex]")
 {
     string input = "";
-    REQUIRE(tokenize(input).empty());
+    REQUIRE(lex(input).empty());
 }
 
-TEST_CASE("single-char tokens", "[tokenize]")
+TEST_CASE("single-char tokens", "[lex]")
 {
     string input = "t()+-*/%&|^";
     vector<string> expected = {"t", "(", ")", "+", "-", "*", "/", "%", "&", "|", "^"};
-    REQUIRE(tokenize(input) == expected);
+    REQUIRE(lex(input) == expected);
 }
 
-TEST_CASE("whitespace", "[tokenize]")
+TEST_CASE("whitespace", "[lex]")
 {
     string input = " ( t ) ";
     vector<string> expected = {"(", "t", ")"};
-    REQUIRE(tokenize(input) == expected);
+    REQUIRE(lex(input) == expected);
 }
 
-TEST_CASE("integers", "[tokenize]")
+TEST_CASE("integers", "[lex]")
 {
     string input = "(t+1)";
     vector<string> expected = {"(", "t", "+", "1", ")"};
-    REQUIRE(tokenize(input) == expected);
+    REQUIRE(lex(input) == expected);
 }
 
-TEST_CASE("multi-digit integers", "[tokenize]")
+TEST_CASE("multi-digit integers", "[lex]")
 {
     string input = "t&63";
     vector<string> expected = {"t", "&", "63"};
-    REQUIRE(tokenize(input) == expected);
+    REQUIRE(lex(input) == expected);
 }
 
-TEST_CASE("multi-char tokens", "[tokenize]")
+TEST_CASE("multi-char tokens", "[lex]")
 {
     string input = "t>>1";
     vector<string> expected = {"t", ">>", "1"};
-    REQUIRE(tokenize(input) == expected);
+    REQUIRE(lex(input) == expected);
 }
 
-TEST_CASE("tokenize crowd", "[tokenize]")
+TEST_CASE("lex crowd", "[lex]")
 {
     std::string input = "((t<<1)^((t<<1)+(t>>7)&t>>12))|t>>(4-(1^7&(t>>19)))|t>>7";
     std::vector<std::string> expected = {
@@ -97,5 +97,5 @@ TEST_CASE("tokenize crowd", "[tokenize]")
         ">>",
         "7",
     };
-    REQUIRE(bb::tokenize(input) == expected);
+    REQUIRE(bb::lex(input) == expected);
 }