diff --git a/preprocess.lua b/preprocess.lua index 7d0673d..9ddffc0 100644 --- a/preprocess.lua +++ b/preprocess.lua @@ -123,7 +123,7 @@ -local PP_VERSION = "1.15.0" +local PP_VERSION = "1.15.0-dev" local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString(). @@ -733,6 +733,21 @@ function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax) tok = {type="pp_keyword", representation=repr, value=word} end + -- Preprocessor symbol. + elseif s:find("^%$", ptr) then + if not allowPpTokens then + errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor symbol. (Feature not enabled.)") + end + + local i1, i2, repr, word = s:find("^(%$([%a_][%w_]*))", ptr) + if not i1 then + errorInFile(s, path, ptr+1, "Tokenizer", "Expected an identifier.") + elseif KEYWORDS[word] then + errorInFile(s, path, ptr+1, "Tokenizer", "Invalid preprocessor symbol '%s'. (Must not be a Lua keyword.)", word) + end + ptr = i2+1 + tok = {type="pp_symbol", representation=repr, value=word} + else errorInFile(s, path, ptr, "Tokenizer", "Unknown character.") end @@ -1629,6 +1644,7 @@ local numberFormatters = { -- whitespaceToken = newToken( "whitespace", contents ) -- ppEntryToken = newToken( "pp_entry", isDouble ) -- ppKeywordToken = newToken( "pp_keyword", ppKeyword ) -- ppKeyword can be "@". +-- ppKeywordToken = newToken( "pp_symbol", identifier ) -- -- commentToken = { type="comment", representation=string, value=string, long=isLongForm } -- identifierToken = { type="identifier", representation=string, value=string } @@ -1639,6 +1655,7 @@ local numberFormatters = { -- whitespaceToken = { type="whitespace", representation=string, value=string } -- ppEntryToken = { type="pp_entry", representation=string, value=string, double=isDouble } -- ppKeywordToken = { type="pp_keyword", representation=string, value=string } +-- ppSymbolToken = { type="pp_symbol", representation=string, value=string } -- -- Number formats: -- "integer" E.g. 42 @@ -1684,6 +1701,8 @@ function metaFuncs.newToken(tokType, ...) error("Identifier length is 0.", 2) elseif not ident:find"^[%a_][%w_]*$" then errorf(2, "Bad identifier format: '%s'", ident) + elseif KEYWORDS[ident] then + errorf(2, "Identifier must not be a keyword: '%s'", ident) end return {type="identifier", representation=ident, value=ident} @@ -1780,6 +1799,20 @@ function metaFuncs.newToken(tokType, ...) return {type="pp_keyword", representation="@"..keyword, value=keyword} end + elseif tokType == "pp_symbol" then + local ident = ... + assertarg(2, ident, "string") + + if ident == "" then + error("Identifier length is 0.", 2) + elseif not ident:find"^[%a_][%w_]*$" then + errorf(2, "Bad identifier format: '%s'", ident) + elseif KEYWORDS[ident] then + errorf(2, "Identifier must not be a keyword: '%s'", ident) + else + return {type="pp_symbol", representation="$"..ident, value=ident} + end + else errorf(2, "Invalid token type '%s'.", tostring(tokType)) end @@ -1812,6 +1845,13 @@ function metaEnv.__ASSERTLUA(lua) return lua end +function metaEnv.__EVALSYMBOL(v) + if type(v) == "function" then + v = v() + end + return v +end + local function getLineCountWithCode(tokens) @@ -1861,6 +1901,7 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats) -- @file -- @line -- ` ... ` + -- $symbol -- if not stats.hasPreprocessorCode then return tokensToExpand @@ -1903,6 +1944,20 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats) tableInsert(tokens, stringTok) tableRemove(tokenStack) -- the string + -- Symbol. (Should this expand later? Does it matter?) + elseif isToken(tok, "pp_symbol") then + local ppSymbolTok = tok + + -- $symbol + tableRemove(tokenStack) -- '$symbol' + tableInsert(tokens, newTokenAt({type="pp_entry", value="!!", representation="!!", double=true}, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="identifier", value="__EVALSYMBOL", representation="__EVALSYMBOL" }, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="identifier", value=ppSymbolTok.value, representation=ppSymbolTok.value}, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok)) + tableInsert(tokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok)) + -- Anything else. else tableInsert(tokens, tok) @@ -2549,7 +2604,7 @@ local function _processFileOrString(params, isFile) } for _, tok in ipairs(tokens) do - if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") then + if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") or isToken(tok, "pp_symbol") then stats.hasPreprocessorCode = true stats.hasMetaprogram = true break @@ -2857,6 +2912,9 @@ local function _processFileOrString(params, isFile) tableInsert(metaParts, metaBlock) tableInsert(metaParts, "))\n") + elseif metaBlock:find(",", 1, true) and loadLuaString("return'',"..metaBlock) then + errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Ambiguous preprocessor block contents. (Comma-separated lists are not supported here.)") + elseif doOutputLua then -- We could do something other than error here. Room for more functionality. errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Preprocessor block does not contain a valid value expression.") diff --git a/tests/quickTest.lua2p b/tests/quickTest.lua2p index 7e3cf32..2358100 100644 --- a/tests/quickTest.lua2p +++ b/tests/quickTest.lua2p @@ -144,6 +144,17 @@ local n2 = @@ADD1!!("43-2") +-- Symbols. +!local RANDOM = "math.random()" +local rand = $RANDOM + +!local function EQUATION() return "x*3+y" end +local x = 5 +local y = 89 +local z = $EQUATION + + + -- Misc. print(!("dataFromCommandLine: "..tostring(dataFromCommandLine))) print(!(("This file and line: %s:%d"):format(@file, @line))) diff --git a/tests/quickTest.output.lua b/tests/quickTest.output.lua index 41c90d9..acb5b40 100644 --- a/tests/quickTest.output.lua +++ b/tests/quickTest.output.lua @@ -99,6 +99,19 @@ local d = {1+2} local n = 58 +local n1 = 41+1 +local n2 = 43-2+1 + + + +-- Symbols + +local rand = math.random() + +local x = 5 +local y = 89 +local z = x*3+y + -- Misc. diff --git a/tests/suite.lua b/tests/suite.lua index 22c26a8..e3291ea 100644 --- a/tests/suite.lua +++ b/tests/suite.lua @@ -176,6 +176,13 @@ doTest("Expression or not?", function() !( x = math.floor(1.5) ) ]]}) assertCodeOutput(luaOut, [[]]) + + -- Invalid: Comma-separated expressions are ambiguous. + assert(not pp.processString{ code=[[ x = !(1, 2) ]]}) + assert(not pp.processString{ code=[[ x = !!("a", "b") ]]}) + + -- Invalid: !!() must always have an expression. + assert(not pp.processString{ code=[[ !!( x = y ) ]]}) end) doTest("Output values of different types", function() @@ -364,6 +371,32 @@ doTest("Macros", function() assert(not pp.processString{ code=[[ !(function ECHO(v) return v end) v = @@ECHO( !!( !(1) ) ) ]]}) end) +doTest("Preprocessor symbols", function() + local pp = ppChunk() + + local luaOut = assert(pp.processString{ code=[[ + !local FOO = "y" + x = $FOO + ]]}) + assertCodeOutput(luaOut, [[x = y]]) + + local luaOut = assert(pp.processString{ code=[[ + !local function FOO() return "y" end + x = $FOO + ]]}) + assertCodeOutput(luaOut, [[x = y]]) + + -- Invalid: Symbols must result in strings. + assert(not pp.processString{ code=[[ + !local BAD = 840 + v = $BAD + ]]}) + assert(not pp.processString{ code=[[ + !local function BAD() return 840 end + v = $BAD + ]]}) +end) + addLabel("Library API") @@ -389,6 +422,8 @@ doTest("Create tokens", function() -- Identifier. assertToken(pp.newToken("identifier", "foo"), "identifier", "foo", "foo", nil, nil) + assert(not pcall(pp.newToken, "identifier", "if")) + -- Keyword. assertToken(pp.newToken("keyword", "if"), "keyword", "if", "if", nil, nil) @@ -440,6 +475,13 @@ doTest("Create tokens", function() assertToken(pp.newToken("pp_keyword", "@" ), "pp_keyword", "insert", "@@", nil, nil) assert(not pcall(pp.newToken, "pp_keyword", "bad")) + + -- Preprocessor symbol. + assertToken(pp.newToken("pp_symbol", "foo"), "pp_symbol", "foo", "$foo", nil, nil) + + assert(not pcall(pp.newToken, "pp_symbol", "")) + assert(not pcall(pp.newToken, "pp_symbol", "if")) + assert(not pcall(pp.newToken, "pp_symbol", "$foo")) end) doTest("Get useful tokens", function()