refactored ParserAST

midori/src/main.cpp

@@ -22,7 +22,7 @@ int test_generator() {
 	std::fstream f("../src/parser.txt", std::fstream::in);
 	std::unique_ptr<Parser> p = ParserGenerator::from_file(&f);
 	std::stringstream ss;
-	ss << "a1ab2bc3cd4d";
+	ss << "a1ab2b \tc3cd4d";
 	FileInputStream fis(&ss);
 	std::unique_ptr<MatchedNonterminal> m = p->parse(&fis);
 	assert(m != nullptr);
@@ -32,7 +32,7 @@ int test_generator() {
 int main() {
 	ULong x = ~0;
 	std::cout << "-1 is " << x << std::endl;
-	test_generator();
 	test_regex_engine();
+	test_generator();
 	return 0;
 }

midori/src/midori/generator.cpp

@@ -3,28 +3,22 @@
 #include "regex_engine.h"
 #include <vector>
 
-class ParserASTStringList : public ParserAST {
-public:
-	std::vector<std::string> list;
-	ParserASTStringList(std::vector<std::string> list) : list(list) {
-		return;
-	}
-};
-
-class ParserASTStringListList : public ParserAST {
-public:
-	std::vector<std::vector<std::string>> list;
-	ParserASTStringListList(std::vector<std::vector<std::string>> list) : list(list) {
-		return;
-	}
-};
+typedef std::vector<std::string> StringList;
+typedef std::vector<StringList> StringListList;
+typedef ParserValue<StringList> ParserValueStringList;
+typedef ParserValue<StringListList> ParserValueStringListList;
 
 std::unique_ptr<Parser> ParserGenerator::from_file(std::istream* is) {
 	RegexEngine re;
 	std::unique_ptr<Parser> ret(new Parser);
 	std::unique_ptr<Parser> p(new Parser);
+	std::string start;
+
 	p->add_token("WHITESPACE", re.parse("[ \\t]"));
 	p->add_token("NL", re.parse("\\n"));
+	p->add_token("PERCENT", re.parse("%"));
+	p->add_token("START", re.parse("start"));
+	p->add_token("SKIP", re.parse("skip"));
 	p->add_token("TOKEN", re.parse("[A-Z][a-zA-Z0-9_]*"));
 	p->add_token("REGEX", re.parse("/[^\\n]+"));
 	p->add_token("NONTERMINAL", re.parse("[a-z][a-zA-Z0-9_]*"));
@@ -39,10 +33,23 @@ std::unique_ptr<Parser> ParserGenerator::from_file(std::istream* is) {
 	p->add_production("contents", { "line", "NL", "contents" }, nullptr);
 	p->add_production("contents", { "line" }, nullptr);
 
+	p->add_production("line", { "line_start" }, nullptr);
+	p->add_production("line", { "line_skip" }, nullptr);
 	p->add_production("line", { "line_token" }, nullptr);
 	p->add_production("line", { "line_production" }, nullptr);
 	p->add_production("line", {}, nullptr);
 
+	p->add_production("line_start", { "PERCENT", "START", "COLON", "NONTERMINAL" }, [ &start ](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
+		start = m->terminal(3)->token->lexeme;
+		return nullptr;
+	});
+
+	p->add_production("line_skip", { "PERCENT", "SKIP", "COLON", "TOKEN" }, [ &ret ](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
+		std::string skip = m->terminal(3)->token->lexeme;
+		ret->add_skip(skip);
+		return nullptr;
+	});
+
 	p->add_production("line_token", { "TOKEN", "COLON", "REGEX" }, [ &ret, &re ](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		std::string t = m->terminal(0)->token->lexeme;
 		std::string r = m->terminal(2)->token->lexeme.substr(1);
@@ -55,30 +62,29 @@ std::unique_ptr<Parser> ParserGenerator::from_file(std::istream* is) {
 		return nullptr;
 	});
 
-	p->add_production("line_production", { "NONTERMINAL", "COLON", "production_list", "SEMICOLON" }, [ &ret ](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
+	p->add_production("nl_optional", { "NL" }, nullptr);
+	p->add_production("nl_optional", {}, nullptr);
+
+	p->add_production("line_production", { "NONTERMINAL", "nl_optional", "COLON", "production_list", "nl_optional", "SEMICOLON" }, [ &ret ](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		std::string target = m->terminal(0)->token->lexeme;
-		std::unique_ptr<ParserAST> n = std::move(m->nonterminal(2)->value);
-		ParserASTStringListList* l = dynamic_cast<ParserASTStringListList*>(n.get());
-		for (std::vector<std::string> const& p : l->list) {
+		StringListList& l = m->nonterminal(3)->value->get<StringListList>();
+		for (std::vector<std::string> const& p : l) {
 			ret->add_production(target, p, nullptr);
 		}
 		return nullptr;
 	});
 
-	p->add_production("nl_optional", { "NL" }, nullptr);
-	p->add_production("nl_optional", {}, nullptr);
-
 	p->add_production("production_list", { "production_list", "nl_optional", "BAR", "production" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		std::unique_ptr<ParserAST> n1 = std::move(m->nonterminal(0)->value);
 		std::unique_ptr<ParserAST> n2 = std::move(m->nonterminal(3)->value);
-		ParserASTStringListList* l = dynamic_cast<ParserASTStringListList*>(n1.get());
-		l->list.push_back(dynamic_cast<ParserASTStringList*>(n2.get())->list);
+		StringListList& l = n1->get<StringListList>();
+		l.push_back(n2->get<StringList>());
 		return n1;
 	});
 	p->add_production("production_list", { "nl_optional", "production" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		std::unique_ptr<ParserAST> n = std::move(m->nonterminal(1)->value);
-		ParserASTStringList* l = dynamic_cast<ParserASTStringList*>(n.get());
-		return std::unique_ptr<ParserAST>(new ParserASTStringListList({ l->list }));
+		StringList& l = n->get<StringList>();
+		return std::unique_ptr<ParserAST>(new ParserValueStringListList({ l }));
 	});
 	/*
 	 * TODO: why doesn't the following work?
@@ -95,31 +101,35 @@ std::unique_ptr<Parser> ParserGenerator::from_file(std::istream* is) {
 	});
 	p->add_production("production", { "EPSILON" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		(void) m;
-		return std::unique_ptr<ParserAST>(new ParserASTStringList({}));
+		return std::unique_ptr<ParserAST>(new ParserValueStringList({}));
 	});
 
 	p->add_production("symbol_list", { "symbol_list", "symbol" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
 		std::unique_ptr<ParserAST> n = std::move(m->nonterminal(0)->value);
-		ParserASTStringList* l = dynamic_cast<ParserASTStringList*>(n.get());
-		ParserASTString* s = dynamic_cast<ParserASTString*>(m->nonterminal(1)->value.get());
-		l->list.push_back(s->str);
+		StringList& l = n->get<StringList>();
+		std::string str = m->nonterminal(1)->value->get<std::string>();
+		l.push_back(str);
 		return n;
 	});
 	p->add_production("symbol_list", { "symbol" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
-		ParserASTString* s = dynamic_cast<ParserASTString*>(m->nonterminal(0)->value.get());
-		return std::unique_ptr<ParserAST>(new ParserASTStringList({ s->str }));
+		std::string str = m->nonterminal(0)->value->get<std::string>();
+		return std::unique_ptr<ParserAST>(new ParserValueStringList({ str }));
 	});
 
 	p->add_production("symbol", { "TOKEN" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
-		return std::unique_ptr<ParserAST>(new ParserASTString(m->terminal(0)->token->lexeme));
+		return std::unique_ptr<ParserAST>(new ParserValue<std::string>(m->terminal(0)->token->lexeme));
 	});
 	p->add_production("symbol", { "NONTERMINAL" }, [](MatchedNonterminal* m) -> std::unique_ptr<ParserAST> {
-		return std::unique_ptr<ParserAST>(new ParserASTString(m->terminal(0)->token->lexeme));
+		return std::unique_ptr<ParserAST>(new ParserValue<std::string>(m->terminal(0)->token->lexeme));
 	});
 
 	p->generate(Parser::Type::LALR1, "grammar");
 	FileInputStream fis(is);
 	std::unique_ptr<MatchedNonterminal> m = p->parse(&fis);
-	ret->generate(Parser::Type::LALR1, "root");
+	if (start.length() == 0) {
+		std::cout << "No start rule" << std::endl;
+		return nullptr;
+	}
+	ret->generate(Parser::Type::LALR1, start);
 	return ret;
 }

midori/src/midori/lexer.h

@@ -6,6 +6,7 @@
 #include <vector>
 #include <istream>
 #include <functional>
+#include <memory>
 #include "global.h"
 #include "regex_ast.h"
 

midori/src/midori/parser.cpp

@@ -8,10 +8,6 @@ ParserAST::~ParserAST() {
 	return;
 }
 
-ParserASTString::ParserASTString(std::string s) : str(s) {
-	return;
-}
-
 Match::~Match() {
 	return;
 }

midori/src/midori/parser.h

@@ -14,19 +14,29 @@
 class ParserAST;
 class MatchedNonterminal;
 struct Production;
+template <typename T> class ParserValue;
 
 typedef std::function<std::unique_ptr<ParserAST>(MatchedNonterminal*)> ProductionHandler;
 typedef std::function<std::unique_ptr<MatchedNonterminal>(std::unique_ptr<MatchedNonterminal>)> RewriteHandler;
 
+
 class ParserAST {
 public:
 	virtual ~ParserAST() = 0;
+
+	template <typename T> T& get() {
+		// TODO: why does this compile?
+		return dynamic_cast<ParserValue<T>*>(this)->value;
+	}
 };
 
-class ParserASTString : public ParserAST {
+template <typename T> class ParserValue : public ParserAST {
 public:
-	std::string str;
-	ParserASTString(std::string);
+	T value;
+
+	ParserValue(T v) : value(std::move(v)) {
+		return;
+	}
 };
 
 struct Production {