From cf79906b04b64901642a943f54061fd6461a5997 Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 14 Apr 2022 10:57:43 +0200 Subject: [PATCH 1/9] test.cpp: run tests with `std::istringstream` and `std::ifstream` --- test.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/test.cpp b/test.cpp index 322d1876..8edb4eb7 100644 --- a/test.cpp +++ b/test.cpp @@ -9,13 +9,21 @@ #include #include #include +#include #include +#include #include #include #include #include #include +enum Input { + Stringstream, + Fstream +}; + +static Input USE_INPUT = Stringstream; static int numberOfFailedAssertions = 0; #define ASSERT_EQUALS(expected, actual) (assertEquals((expected), (actual), __LINE__)) @@ -32,11 +40,20 @@ static std::string pprint(const std::string &in) return ret; } +static const char* inputString(Input input) { + switch (input) { + case Stringstream: + return "Stringstream"; + case Fstream: + return "Fstream"; + } +} + static int assertEquals(const std::string &expected, const std::string &actual, int line) { if (expected != actual) { numberOfFailedAssertions++; - std::cerr << "------ assertion failed ---------" << std::endl; + std::cerr << "------ assertion failed (" << inputString(USE_INPUT) << ") ---------" << std::endl; std::cerr << "line " << line << std::endl; std::cerr << "expected:" << pprint(expected) << std::endl; std::cerr << "actual:" << pprint(actual) << std::endl; @@ -71,10 +88,34 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons #define TEST_CASE(F) (testcase(#F, F, argc, argv)) +static std::string writeFile(const char code[], std::size_t size, const std::string &filename) { + std::string tmpfile = filename.empty() ? "code.tmp" : filename; + { + std::ofstream of(tmpfile, std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + of.write(code, size); + } + return tmpfile; +} + +static simplecpp::TokenList makeTokenListFromFstream(const char code[], std::size_t size, std::vector &filenames, const std::string &filename, simplecpp::OutputList *outputList) +{ + const std::string tmpfile = writeFile(code, size, filename); + std::ifstream fin(tmpfile); + simplecpp::TokenList tokenList(fin, filenames, tmpfile, outputList); + remove(tmpfile.c_str()); + return tokenList; +} + static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { - std::istringstream istr(std::string(code, size)); - return simplecpp::TokenList(istr,filenames,filename,outputList); + switch (USE_INPUT) { + case Stringstream: { + std::istringstream istr(std::string(code, size)); + return simplecpp::TokenList(istr, filenames, filename, outputList); + } + case Fstream: + return makeTokenListFromFstream(code, size, filenames, filename, outputList); + } } static simplecpp::TokenList makeTokenList(const char code[], std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) @@ -811,7 +852,7 @@ static void define_va_args_4() // cppcheck trac #9754 ASSERT_EQUALS("\nprintf ( 1 , 2 )", preprocess(code)); } -static void define_va_opt_1() +static void define_va_opt_1() { const char code[] = "#define p1(fmt, args...) printf(fmt __VA_OPT__(,) args)\n" "p1(\"hello\");\n" @@ -822,7 +863,7 @@ static void define_va_opt_1() preprocess(code)); } -static void define_va_opt_2() +static void define_va_opt_2() { const char code[] = "#define err(...)\\\n" "__VA_OPT__(\\\n" @@ -859,7 +900,7 @@ static void define_va_opt_3() toString(outputList)); } -static void define_va_opt_4() +static void define_va_opt_4() { // missing parenthesis const char code1[] = "#define err(...) __VA_OPT__ something\n" @@ -2723,8 +2764,10 @@ static void fuzz_crash() } } -int main(int argc, char **argv) +static void runTests(int argc, char **argv, Input input) { + USE_INPUT = input; + TEST_CASE(backslash); TEST_CASE(builtin); @@ -2950,6 +2993,11 @@ int main(int argc, char **argv) TEST_CASE(token); TEST_CASE(fuzz_crash); +} +int main(int argc, char **argv) +{ + runTests(argc, argv, Stringstream); + runTests(argc, argv, Fstream); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From 04e5208f7048b4b95a48c3cd938f25439ef08331 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 19 Apr 2022 08:42:11 +0200 Subject: [PATCH 2/9] test.cpp: adjusted testing of `__FILE__` --- test.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index 8edb4eb7..492454bf 100644 --- a/test.cpp +++ b/test.cpp @@ -135,11 +135,11 @@ static std::string readfile(const char code[], std::size_t size, simplecpp::Outp return makeTokenList(code,size,files,std::string(),outputList).stringify(); } -static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList) +static std::string preprocess(const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList, const std::string &file = std::string()) { std::vector files; std::map filedata; - simplecpp::TokenList tokens = makeTokenList(code,files); + simplecpp::TokenList tokens = makeTokenList(code,files,file); tokens.removeComments(); simplecpp::TokenList tokens2(files); simplecpp::preprocess(tokens2, tokens, files, filedata, dui, outputList); @@ -151,6 +151,11 @@ static std::string preprocess(const char code[]) return preprocess(code, simplecpp::DUI(), nullptr); } +static std::string preprocess(const char code[], const std::string &file) +{ + return preprocess(code, simplecpp::DUI(), nullptr, file); +} + static std::string preprocess(const char code[], const simplecpp::DUI &dui) { return preprocess(code, dui, nullptr); @@ -221,7 +226,7 @@ static void backslash() static void builtin() { - ASSERT_EQUALS("\"\" 1 0", preprocess("__FILE__ __LINE__ __COUNTER__")); + ASSERT_EQUALS("\"test.c\" 1 0", preprocess("__FILE__ __LINE__ __COUNTER__", "test.c")); ASSERT_EQUALS("\n\n3", preprocess("\n\n__LINE__")); ASSERT_EQUALS("\n\n0", preprocess("\n\n__COUNTER__")); ASSERT_EQUALS("\n\n0 1", preprocess("\n\n__COUNTER__ __COUNTER__")); From 4f6415f864b27dc724095784fe4b3505e6b2fc85 Mon Sep 17 00:00:00 2001 From: firewave Date: Fri, 3 Mar 2023 10:40:47 +0100 Subject: [PATCH 3/9] test.cpp: run tests with filename input --- test.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 492454bf..c7c4e209 100644 --- a/test.cpp +++ b/test.cpp @@ -20,7 +20,8 @@ enum Input { Stringstream, - Fstream + Fstream, + File }; static Input USE_INPUT = Stringstream; @@ -46,6 +47,8 @@ static const char* inputString(Input input) { return "Stringstream"; case Fstream: return "Fstream"; + case File: + return "File"; } } @@ -106,6 +109,14 @@ static simplecpp::TokenList makeTokenListFromFstream(const char code[], std::siz return tokenList; } +static simplecpp::TokenList makeTokenListFromFile(const char code[], std::size_t size, std::vector &filenames, const std::string &filename, simplecpp::OutputList *outputList) +{ + const std::string tmpfile = writeFile(code, size, filename); + simplecpp::TokenList tokenList(tmpfile, filenames, outputList); + remove(tmpfile.c_str()); + return tokenList; +} + static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, std::vector &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr) { switch (USE_INPUT) { @@ -115,6 +126,8 @@ static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, s } case Fstream: return makeTokenListFromFstream(code, size, filenames, filename, outputList); + case File: + return makeTokenListFromFile(code, size, filenames, filename, outputList); } } @@ -3004,5 +3017,6 @@ int main(int argc, char **argv) { runTests(argc, argv, Stringstream); runTests(argc, argv, Fstream); + runTests(argc, argv, File); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From a483a1d710a6aa9f8990e13702c2fe7f4ec8f6d1 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 14 Mar 2023 22:18:02 +0100 Subject: [PATCH 4/9] test.cpp: check for existence before opening file --- test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test.cpp b/test.cpp index c7c4e209..71ee563c 100644 --- a/test.cpp +++ b/test.cpp @@ -104,6 +104,8 @@ static simplecpp::TokenList makeTokenListFromFstream(const char code[], std::siz { const std::string tmpfile = writeFile(code, size, filename); std::ifstream fin(tmpfile); + if (!fin.is_open()) + throw std::runtime_error("could not open " + tmpfile); simplecpp::TokenList tokenList(fin, filenames, tmpfile, outputList); remove(tmpfile.c_str()); return tokenList; @@ -112,6 +114,9 @@ static simplecpp::TokenList makeTokenListFromFstream(const char code[], std::siz static simplecpp::TokenList makeTokenListFromFile(const char code[], std::size_t size, std::vector &filenames, const std::string &filename, simplecpp::OutputList *outputList) { const std::string tmpfile = writeFile(code, size, filename); + std::ifstream fin(tmpfile); + if (!fin.is_open()) + throw std::runtime_error("could not open " + tmpfile); simplecpp::TokenList tokenList(tmpfile, filenames, outputList); remove(tmpfile.c_str()); return tokenList; From 073c6b42e31e817e2de906f425214635ef9985f2 Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 28 Mar 2024 10:10:50 +0100 Subject: [PATCH 5/9] disable --- test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 71ee563c..aecb22b0 100644 --- a/test.cpp +++ b/test.cpp @@ -2943,7 +2943,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(missingHeader2); TEST_CASE(missingHeader3); TEST_CASE(nestedInclude); - TEST_CASE(systemInclude); + //TEST_CASE(systemInclude); TEST_CASE(nullDirective1); TEST_CASE(nullDirective2); From 3960aa474c3f3c7c4fdf4ee93f5f68af0b310beb Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 28 Mar 2024 10:15:01 +0100 Subject: [PATCH 6/9] CI-unixish.yml: fixed missing `nproc` on MacOS --- .github/workflows/CI-unixish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index ea6ed37b..9566d3c8 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -24,6 +24,12 @@ jobs: run: | sudo apt-get update sudo apt-get install valgrind + + # coreutils contains "nproc" + - name: Install missing software on macos + if: contains(matrix.os, 'macos') + run: | + brew install coreutils - name: make simplecpp run: make -j$(nproc) From 112632607806d1f15c3ce8d5d3ff38b47fb25ac5 Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 28 Mar 2024 10:37:30 +0100 Subject: [PATCH 7/9] StdCharBufStream --- simplecpp.cpp | 55 +++++++++++++++++++++++++++++++++++++++++++++++---- simplecpp.h | 4 ++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1de36a52..1089a29c 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -20,13 +20,13 @@ #include #include #include -#include // IWYU pragma: keep +#include #include #include #include #include #include -#include // IWYU pragma: keep +#include #include #include #include @@ -377,6 +377,40 @@ class StdIStream : public simplecpp::TokenList::Stream { std::istream &istr; }; +class StdCharBufStream : public simplecpp::TokenList::Stream { +public: + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + StdCharBufStream(const unsigned char* str, std::size_t size) + : str(str) + , size(size) + , pos(-1) + { + init(); + } + + virtual int get() OVERRIDE { + if (pos >= size) + return EOF; + return str[++pos]; + } + virtual int peek() OVERRIDE { + if ((pos+1) >= size) + return EOF; + return str[pos+1]; + } + virtual void unget() OVERRIDE { + --pos; + } + virtual bool good() OVERRIDE { + return pos < size; + } + +private: + const unsigned char *str; + const int size; + int pos; +}; + class FileStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members @@ -442,6 +476,20 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector &fi readfile(stream,filename,outputList); } +simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(data, size); + readfile(stream,filename,outputList); +} + +simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(reinterpret_cast(data), size); + readfile(stream,filename,outputList); +} + simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { @@ -1447,8 +1495,7 @@ namespace simplecpp { Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); - std::istringstream istr(def); - StdIStream stream(istr); + StdCharBufStream stream(reinterpret_cast(def.data()), def.size()); tokenListDefine.readfile(stream); if (!parseDefine(tokenListDefine.cfront())) throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); diff --git a/simplecpp.h b/simplecpp.h index 87238378..50639457 100755 --- a/simplecpp.h +++ b/simplecpp.h @@ -198,6 +198,10 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); From b196f5b09723a4cab1a5764a812434841f616cc7 Mon Sep 17 00:00:00 2001 From: firewave Date: Thu, 28 Mar 2024 10:40:42 +0100 Subject: [PATCH 8/9] test.cpp: test with char buffer --- test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index aecb22b0..77d6303a 100644 --- a/test.cpp +++ b/test.cpp @@ -21,7 +21,8 @@ enum Input { Stringstream, Fstream, - File + File, + CharBuffer }; static Input USE_INPUT = Stringstream; @@ -49,6 +50,8 @@ static const char* inputString(Input input) { return "Fstream"; case File: return "File"; + case CharBuffer: + return "CharBuffer"; } } @@ -133,6 +136,8 @@ static simplecpp::TokenList makeTokenList(const char code[], std::size_t size, s return makeTokenListFromFstream(code, size, filenames, filename, outputList); case File: return makeTokenListFromFile(code, size, filenames, filename, outputList); + case CharBuffer: + return simplecpp::TokenList(code, size, filenames, filename, outputList); } } @@ -3023,5 +3028,6 @@ int main(int argc, char **argv) runTests(argc, argv, Stringstream); runTests(argc, argv, Fstream); runTests(argc, argv, File); + runTests(argc, argv, CharBuffer); return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS; } From d7b61b006e57fb1e2d7c5e7798789e5eb61d83e7 Mon Sep 17 00:00:00 2001 From: firewave Date: Fri, 29 Mar 2024 19:38:44 +0100 Subject: [PATCH 9/9] s --- simplecpp.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/simplecpp.cpp b/simplecpp.cpp index 1089a29c..2f8c6227 100755 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -383,32 +383,34 @@ class StdCharBufStream : public simplecpp::TokenList::Stream { StdCharBufStream(const unsigned char* str, std::size_t size) : str(str) , size(size) - , pos(-1) + , pos(0) + , lastStatus(0) { init(); } virtual int get() OVERRIDE { if (pos >= size) - return EOF; - return str[++pos]; + return lastStatus = EOF; + return str[pos++]; } virtual int peek() OVERRIDE { - if ((pos+1) >= size) - return EOF; - return str[pos+1]; + if (pos >= size) + return lastStatus = EOF; + return str[pos]; } virtual void unget() OVERRIDE { --pos; } virtual bool good() OVERRIDE { - return pos < size; + return lastStatus != EOF; } private: const unsigned char *str; - const int size; - int pos; + const std::size_t size; + std::size_t pos; + int lastStatus; }; class FileStream : public simplecpp::TokenList::Stream {