Added preprocessor symbols ($identifier).

Detecting errors such as !(x,y).

preprocess.lua

@@ -123,7 +123,7 @@
 
 
 
-local PP_VERSION = "1.15.0"
+local PP_VERSION = "1.15.0-dev"
 
 local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString().
 
@@ -733,6 +733,21 @@ function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax)
 				tok = {type="pp_keyword", representation=repr, value=word}
 			end
 
+		-- Preprocessor symbol.
+		elseif s:find("^%$", ptr) then
+			if not allowPpTokens then
+				errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor symbol. (Feature not enabled.)")
+			end
+
+			local i1, i2, repr, word = s:find("^(%$([%a_][%w_]*))", ptr)
+			if not i1 then
+				errorInFile(s, path, ptr+1, "Tokenizer", "Expected an identifier.")
+			elseif KEYWORDS[word] then
+				errorInFile(s, path, ptr+1, "Tokenizer", "Invalid preprocessor symbol '%s'. (Must not be a Lua keyword.)", word)
+			end
+			ptr = i2+1
+			tok = {type="pp_symbol", representation=repr, value=word}
+
 		else
 			errorInFile(s, path, ptr, "Tokenizer", "Unknown character.")
 		end
@@ -1629,6 +1644,7 @@ local numberFormatters = {
 --   whitespaceToken  = newToken( "whitespace",  contents )
 --   ppEntryToken     = newToken( "pp_entry",    isDouble )
 --   ppKeywordToken   = newToken( "pp_keyword",  ppKeyword ) -- ppKeyword can be "@".
+--   ppKeywordToken   = newToken( "pp_symbol",   identifier )
 --
 --   commentToken     = { type="comment",     representation=string, value=string, long=isLongForm }
 --   identifierToken  = { type="identifier",  representation=string, value=string }
@@ -1639,6 +1655,7 @@ local numberFormatters = {
 --   whitespaceToken  = { type="whitespace",  representation=string, value=string }
 --   ppEntryToken     = { type="pp_entry",    representation=string, value=string, double=isDouble }
 --   ppKeywordToken   = { type="pp_keyword",  representation=string, value=string }
+--   ppSymbolToken    = { type="pp_symbol",   representation=string, value=string }
 --
 -- Number formats:
 --   "integer"      E.g. 42
@@ -1684,6 +1701,8 @@ function metaFuncs.newToken(tokType, ...)
 			error("Identifier length is 0.", 2)
 		elseif not ident:find"^[%a_][%w_]*$" then
 			errorf(2, "Bad identifier format: '%s'", ident)
+		elseif KEYWORDS[ident] then
+			errorf(2, "Identifier must not be a keyword: '%s'", ident)
 		end
 
 		return {type="identifier", representation=ident, value=ident}
@@ -1780,6 +1799,20 @@ function metaFuncs.newToken(tokType, ...)
 			return {type="pp_keyword", representation="@"..keyword, value=keyword}
 		end
 
+	elseif tokType == "pp_symbol" then
+		local ident = ...
+		assertarg(2, ident, "string")
+
+		if ident == "" then
+			error("Identifier length is 0.", 2)
+		elseif not ident:find"^[%a_][%w_]*$" then
+			errorf(2, "Bad identifier format: '%s'", ident)
+		elseif KEYWORDS[ident] then
+			errorf(2, "Identifier must not be a keyword: '%s'", ident)
+		else
+			return {type="pp_symbol", representation="$"..ident, value=ident}
+		end
+
 	else
 		errorf(2, "Invalid token type '%s'.", tostring(tokType))
 	end
@@ -1812,6 +1845,13 @@ function metaEnv.__ASSERTLUA(lua)
 	return lua
 end
 
+function metaEnv.__EVALSYMBOL(v)
+	if type(v) == "function" then
+		v = v()
+	end
+	return v
+end
+
 
 
 local function getLineCountWithCode(tokens)
@@ -1861,6 +1901,7 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats)
 	--   @file
 	--   @line
 	--   ` ... `
+	--   $symbol
 	--
 	if not stats.hasPreprocessorCode then
 		return tokensToExpand
@@ -1903,6 +1944,20 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats)
 			tableInsert(tokens, stringTok)
 			tableRemove(tokenStack) -- the string
 
+		-- Symbol. (Should this expand later? Does it matter?)
+		elseif isToken(tok, "pp_symbol") then
+			local ppSymbolTok = tok
+
+			-- $symbol
+			tableRemove(tokenStack) -- '$symbol'
+			tableInsert(tokens, newTokenAt({type="pp_entry",    value="!!",              representation="!!", double=true}, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="punctuation", value="(",               representation="("              }, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="identifier",  value="__EVALSYMBOL",    representation="__EVALSYMBOL"   }, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="punctuation", value="(",               representation="("              }, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="identifier",  value=ppSymbolTok.value, representation=ppSymbolTok.value}, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="punctuation", value=")",               representation=")"              }, ppSymbolTok))
+			tableInsert(tokens, newTokenAt({type="punctuation", value=")",               representation=")"              }, ppSymbolTok))
+
 		-- Anything else.
 		else
 			tableInsert(tokens, tok)
@@ -2549,7 +2604,7 @@ local function _processFileOrString(params, isFile)
 	}
 
 	for _, tok in ipairs(tokens) do
-		if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") then
+		if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") or isToken(tok, "pp_symbol") then
 			stats.hasPreprocessorCode = true
 			stats.hasMetaprogram      = true
 			break
@@ -2857,6 +2912,9 @@ local function _processFileOrString(params, isFile)
 				tableInsert(metaParts, metaBlock)
 				tableInsert(metaParts, "))\n")
 
+			elseif metaBlock:find(",", 1, true) and loadLuaString("return'',"..metaBlock) then
+				errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Ambiguous preprocessor block contents. (Comma-separated lists are not supported here.)")
+
 			elseif doOutputLua then
 				-- We could do something other than error here. Room for more functionality.
 				errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Preprocessor block does not contain a valid value expression.")

tests/quickTest.lua2p

@@ -144,6 +144,17 @@ local n2 = @@ADD1!!("43-2")
 
 
 
+-- Symbols.
+!local RANDOM = "math.random()"
+local rand    = $RANDOM
+
+!local function EQUATION()  return "x*3+y"  end
+local x = 5
+local y = 89
+local z = $EQUATION
+
+
+
 -- Misc.
 print(!("dataFromCommandLine: "..tostring(dataFromCommandLine)))
 print(!(("This file and line: %s:%d"):format(@file, @line)))

tests/quickTest.output.lua

@@ -99,6 +99,19 @@ local d = {1+2}
 
 local n = 58
 
+local n1 = 41+1
+local n2 = 43-2+1
+
+
+
+-- Symbols
+
+local rand = math.random()
+
+local x = 5
+local y = 89
+local z = x*3+y
+
 
 
 -- Misc.

tests/suite.lua

@@ -176,6 +176,13 @@ doTest("Expression or not?", function()
 		!( x = math.floor(1.5) )
 	]]})
 	assertCodeOutput(luaOut, [[]])
+
+	-- Invalid: Comma-separated expressions are ambiguous.
+	assert(not pp.processString{ code=[[ x = !(1, 2)      ]]})
+	assert(not pp.processString{ code=[[ x = !!("a", "b") ]]})
+
+	-- Invalid: !!() must always have an expression.
+	assert(not pp.processString{ code=[[ !!( x = y ) ]]})
 end)
 
 doTest("Output values of different types", function()
@@ -364,6 +371,32 @@ doTest("Macros", function()
 	assert(not pp.processString{ code=[[  !(function ECHO(v) return v end)  v = @@ECHO( !!( !(1) ) )  ]]})
 end)
 
+doTest("Preprocessor symbols", function()
+	local pp = ppChunk()
+
+	local luaOut = assert(pp.processString{ code=[[
+		!local FOO = "y"
+		x = $FOO
+	]]})
+	assertCodeOutput(luaOut, [[x = y]])
+
+	local luaOut = assert(pp.processString{ code=[[
+		!local function FOO()  return "y"  end
+		x = $FOO
+	]]})
+	assertCodeOutput(luaOut, [[x = y]])
+
+	-- Invalid: Symbols must result in strings.
+	assert(not pp.processString{ code=[[
+		!local BAD = 840
+		v = $BAD
+	]]})
+	assert(not pp.processString{ code=[[
+		!local function BAD()  return 840  end
+		v = $BAD
+	]]})
+end)
+
 
 
 addLabel("Library API")
@@ -389,6 +422,8 @@ doTest("Create tokens", function()
 	-- Identifier.
 	assertToken(pp.newToken("identifier", "foo"), "identifier", "foo", "foo", nil, nil)
 
+	assert(not pcall(pp.newToken, "identifier", "if"))
+
 	-- Keyword.
 	assertToken(pp.newToken("keyword", "if"), "keyword", "if", "if", nil, nil)
 
@@ -440,6 +475,13 @@ doTest("Create tokens", function()
 	assertToken(pp.newToken("pp_keyword", "@"   ), "pp_keyword", "insert", "@@",    nil, nil)
 
 	assert(not pcall(pp.newToken, "pp_keyword", "bad"))
+
+	-- Preprocessor symbol.
+	assertToken(pp.newToken("pp_symbol", "foo"), "pp_symbol", "foo", "$foo", nil, nil)
+
+	assert(not pcall(pp.newToken, "pp_symbol", ""))
+	assert(not pcall(pp.newToken, "pp_symbol", "if"))
+	assert(not pcall(pp.newToken, "pp_symbol", "$foo"))
 end)
 
 doTest("Get useful tokens", function()