Skip to content

Commit

Permalink
Expand - Refactor how editions are propagated through token trees.
Browse files Browse the repository at this point in the history
  • Loading branch information
@thepowersgang @rdrpenguin04
thepowersgang authored and rdrpenguin04 committed Nov 17, 2021
1 parent 6a679b0 commit 26a8dad
Show file tree
Hide file tree
Showing 28 changed files with 169 additions and 90 deletions.
9 changes: 9 additions & 0 deletions src/ast/edition.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,14 @@ namespace AST {
Rust2018,
Rust2021,
};
static inline std::ostream& operator<<(std::ostream& os, const Edition& e) {
switch(e)
{
case Edition::Rust2015: os << "Rust2015"; break;
case Edition::Rust2018: os << "Rust2018"; break;
case Edition::Rust2021: os << "Rust2021"; break;
}
return os;
}

}
8 changes: 4 additions & 4 deletions src/expand/asm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class CLlvmAsmExpander:
::std::unique_ptr<TokenStream> expand(const Span& sp, const ::AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

auto template_text = get_string(sp, lex, crate, mod);
::std::vector<::AST::ExprNode_Asm::ValRef> outputs;
Expand Down Expand Up @@ -203,7 +203,7 @@ class CLlvmAsmExpander:

// Convert this into an AST node and insert as an intepolated expression
::AST::ExprNodeP rv = ::AST::ExprNodeP( new ::AST::ExprNode_Asm { mv$(template_text), mv$(outputs), mv$(inputs), mv$(clobbers), mv$(flags) } );
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
return box$( TTStreamO(sp, ParseState(), TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
}
};

Expand Down Expand Up @@ -236,7 +236,7 @@ class CAsmExpander:
// Stabilisation-path `asm!`

Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

std::vector<std::pair<Span,std::string>> raw_lines;
do {
Expand Down Expand Up @@ -519,7 +519,7 @@ class CAsmExpander:

// Convert this into an AST node and insert as an intepolated expression
::AST::ExprNodeP rv = ::AST::ExprNodeP( new ::AST::ExprNode_Asm2 { mv$(options), mv$(lines), mv$(params) } );
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
return box$( TTStreamO(sp, ParseState(), TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
}
};

Expand Down
4 changes: 2 additions & 2 deletions src/expand/assert.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class CExpander_assert:
{
Token tok;

auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);
lex.parse_state().module = &mod;

// assertion condition
Expand Down Expand Up @@ -84,7 +84,7 @@ class CExpander_assert:

toks.push_back( Token(TOK_BRACE_CLOSE) );

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015, Ident::Hygiene::new_scope(), mv$(toks))) );
}
};

Expand Down
6 changes: 3 additions & 3 deletions src/expand/cfg.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,17 +103,17 @@ class CCfgExpander:
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const ::AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);
lex.parse_state().crate = &crate;
lex.parse_state().module = &mod;
auto attrs = Parse_MetaItem(lex);
DEBUG("cfg!() - " << attrs);

if( check_cfg(sp, attrs) ) {
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree({},TOK_RWORD_TRUE )) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015,{},TOK_RWORD_TRUE )) );
}
else {
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree({},TOK_RWORD_FALSE)) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015,{},TOK_RWORD_FALSE)) );
}
}
};
Expand Down
8 changes: 4 additions & 4 deletions src/expand/concat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class CConcatExpander:
{
Token tok;

auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

::std::string rv;
do {
Expand Down Expand Up @@ -64,7 +64,7 @@ class CConcatExpander:
if( tok.type() != TOK_EOF )
throw ParseError::Unexpected(lex, tok, {TOK_COMMA, TOK_EOF});

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token(TOK_STRING, mv$(rv)))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(tt.get_edition(), Token(TOK_STRING, mv$(rv)))) );
}
};

Expand All @@ -74,7 +74,7 @@ class CConcatIdentsExpander:
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

::std::string rv;

Expand All @@ -91,7 +91,7 @@ class CConcatIdentsExpander:
if( tok.type() != TOK_EOF )
throw ParseError::Unexpected(lex, tok, {TOK_COMMA, TOK_EOF});

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token(TOK_IDENT, Ident(lex.get_hygiene(), RcString::new_interned(rv)))) ) );
return box$( TTStreamO(sp, ParseState(), TokenTree(tt.get_edition(), Token(TOK_IDENT, Ident(lex.get_hygiene(), RcString::new_interned(rv)))) ) );
}
};

Expand Down
6 changes: 3 additions & 3 deletions src/expand/env.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
namespace {
// Read a string out of the input stream
::std::string get_string(const Span& sp, const AST::Crate& crate, AST::Module& mod, const TokenTree& tt) {
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

auto n = Parse_ExprVal(lex);
ASSERT_BUG(sp, n, "No expression returned");
Expand Down Expand Up @@ -43,7 +43,7 @@ class CExpanderEnv:
if( !var_val_cstr ) {
ERROR(sp, E0000, "Environment variable '" << varname << "' not defined");
}
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token(TOK_STRING, ::std::string(var_val_cstr)))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token(TOK_STRING, ::std::string(var_val_cstr)))) );
}
};

Expand Down Expand Up @@ -73,7 +73,7 @@ class CExpanderOptionEnv:
rv.push_back( Token(TOK_STRING, ::std::string(var_val_cstr)) );
rv.push_back( Token(TOK_PAREN_CLOSE) );
}
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree( {}, mv$(rv) )) );
return box$( TTStreamO(sp, ParseState(), TokenTree( AST::Edition::Rust2015, {}, mv$(rv) )) );
}
};

Expand Down
10 changes: 5 additions & 5 deletions src/expand/file_line.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class CExpanderFile:
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token(TOK_STRING, ::std::string(get_top_span(sp)->filename.c_str())))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token(TOK_STRING, ::std::string(get_top_span(sp)->filename.c_str())))) );
}
};

Expand All @@ -35,7 +35,7 @@ class CExpanderLine:
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token((uint64_t)get_top_span(sp)->start_line, CORETYPE_U32))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token((uint64_t)get_top_span(sp)->start_line, CORETYPE_U32))) );
}
};

Expand All @@ -44,15 +44,15 @@ class CExpanderColumn:
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token((uint64_t)get_top_span(sp)->start_ofs, CORETYPE_U32))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token((uint64_t)get_top_span(sp)->start_ofs, CORETYPE_U32))) );
}
};
class CExpanderUnstableColumn:
public ExpandProcMacro
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token((uint64_t)get_top_span(sp)->start_ofs, CORETYPE_U32))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token((uint64_t)get_top_span(sp)->start_ofs, CORETYPE_U32))) );
}
};

Expand All @@ -67,7 +67,7 @@ class CExpanderModulePath:
path_str += "::";
path_str += comp.c_str();
}
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree( Token(TOK_STRING, mv$(path_str)) )) );
return box$( TTStreamO(sp, ParseState(), TokenTree( Token(TOK_STRING, mv$(path_str)) )) );
}
};

Expand Down
6 changes: 3 additions & 3 deletions src/expand/format_args.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -830,7 +830,7 @@ namespace {
toks.push_back( TokenTree(TOK_BRACE_CLOSE) );
toks.push_back( TokenTree(TOK_BRACE_CLOSE) );

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(lex.get_edition(), Ident::Hygiene::new_scope(), mv$(toks))) );
}
}

Expand All @@ -841,7 +841,7 @@ class CFormatArgsExpander:
{
Token tok;

auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);
lex.parse_state().module = &mod;

return expand_format_args(sp, crate, lex, /*add_newline=*/false);
Expand All @@ -855,7 +855,7 @@ class CFormatArgsNlExpander:
{
Token tok;

auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);
lex.parse_state().module = &mod;

return expand_format_args(sp, crate, lex, /*add_newline=*/true);
Expand Down
15 changes: 8 additions & 7 deletions src/expand/include.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class CIncludeExpander:
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

auto path = get_string(sp, lex, crate, mod);
GET_CHECK_TOK(tok, lex, TOK_EOF);
Expand All @@ -88,9 +88,10 @@ class CIncludeExpander:
crate.m_extra_files.push_back(file_path);

try {
ParseState ps(crate.m_edition);
ParseState ps;
ps.module = &mod;
return box$( Lexer(file_path, ps) );
DEBUG("Edition = " << crate.m_edition);
return box$( Lexer(file_path, crate.m_edition, ps) );
}
catch(::std::runtime_error& e)
{
Expand All @@ -105,7 +106,7 @@ class CIncludeBytesExpander:
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

auto path = get_string(sp, lex, crate, mod);
GET_CHECK_TOK(tok, lex, TOK_EOF);
Expand All @@ -122,7 +123,7 @@ class CIncludeBytesExpander:

::std::vector<TokenTree> toks;
toks.push_back(Token(TOK_BYTESTRING, mv$(ss.str())));
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015, Ident::Hygiene::new_scope(), mv$(toks))) );
}
};

Expand All @@ -132,7 +133,7 @@ class CIncludeStrExpander:
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
Token tok;
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

auto path = get_string(sp, lex, crate, mod);
GET_CHECK_TOK(tok, lex, TOK_EOF);
Expand All @@ -149,7 +150,7 @@ class CIncludeStrExpander:

::std::vector<TokenTree> toks;
toks.push_back(Token(TOK_STRING, mv$(ss.str())));
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015, Ident::Hygiene::new_scope(), mv$(toks))) );
}
};

Expand Down
4 changes: 2 additions & 2 deletions src/expand/macro_rules.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ class CMacroRulesExpander:
::std::unique_ptr<TokenStream> expand_ident(const Span& sp, const ::AST::Crate& crate, const RcString& ident, const TokenTree& tt, AST::Module& mod) override
{
DEBUG("Parsing macro_rules! " << ident);
TTStream lex(sp, ParseState(crate.m_edition), tt);
TTStream lex(sp, ParseState(), tt);
auto mac = Parse_MacroRules(lex);
DEBUG("macro_rules! " << mod.path() + ident << " " << &*mac);
mod.add_macro( false, ident, mv$(mac) );

return ::std::unique_ptr<TokenStream>( new TTStreamO(sp, ParseState(crate.m_edition), TokenTree()) );
return ::std::unique_ptr<TokenStream>( new TTStreamO(sp, ParseState(), TokenTree()) );
}
};

Expand Down
2 changes: 1 addition & 1 deletion src/expand/panic.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class CExpander_panic:
}
toks.push_back( Token(TOK_PAREN_CLOSE) );

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(AST::Edition::Rust2015, Ident::Hygiene::new_scope(), mv$(toks))) );
}
};

Expand Down
5 changes: 4 additions & 1 deletion src/expand/proc_macro.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ struct ProcMacroInv:
{
Span m_parent_span;
const ::HIR::ProcMacro& m_proc_macro_desc;
AST::Edition m_edition;
::std::ofstream m_dump_file_out;
::std::ofstream m_dump_file_res;

Expand Down Expand Up @@ -290,6 +291,7 @@ struct ProcMacroInv:

virtual Position getPosition() const override;
virtual Token realGetToken() override;
virtual AST::Edition realGetEdition() const override { return m_edition; }
virtual Ident::Hygiene realGetHygiene() const override;
private:
Token realGetToken_();
Expand Down Expand Up @@ -1113,7 +1115,8 @@ ::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Cra
}

ProcMacroInv::ProcMacroInv(const Span& sp, AST::Edition edition, const char* executable, const ::HIR::ProcMacro& proc_macro_desc):
TokenStream(ParseState(edition)),
TokenStream(ParseState()),
m_edition(edition),
m_parent_span(sp),
m_proc_macro_desc(proc_macro_desc)
{
Expand Down
8 changes: 4 additions & 4 deletions src/expand/rustc_diagnostics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,23 @@ class CExpanderRegisterDiagnostic:
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree()) );
return box$( TTStreamO(sp, ParseState(), TokenTree()) );
}
};
class CExpanderDiagnosticUsed:
public ExpandProcMacro
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree()) );
return box$( TTStreamO(sp, ParseState(), TokenTree()) );
}
};
class CExpanderBuildDiagnosticArray:
public ExpandProcMacro
{
::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const TokenTree& tt, AST::Module& mod) override
{
auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);

Token tok;

Expand Down Expand Up @@ -63,7 +63,7 @@ class CExpanderBuildDiagnosticArray:
toks.push_back( TOK_SQUARE_CLOSE );
toks.push_back( TOK_SEMICOLON );

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree( lex.get_hygiene(), mv$(toks) )) );
return box$( TTStreamO(sp, ParseState(), TokenTree( AST::Edition::Rust2015, lex.get_hygiene(), mv$(toks) )) );
}
};

Expand Down
4 changes: 2 additions & 2 deletions src/expand/stringify.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class CExpander:
Token tok;
::std::string rv;

auto lex = TTStream(sp, ParseState(crate.m_edition), tt);
auto lex = TTStream(sp, ParseState(), tt);
while( GET_TOK(tok, lex) != TOK_EOF )
{
if(!rv.empty())
Expand All @@ -30,7 +30,7 @@ class CExpander:
// TODO: Strip out any `{...}` sequences that aren't from nested
// strings.

return box$( TTStreamO(sp, ParseState(crate.m_edition), TokenTree(Token(TOK_STRING, mv$(rv)))) );
return box$( TTStreamO(sp, ParseState(), TokenTree(Token(TOK_STRING, mv$(rv)))) );
}
};

Expand Down
Loading

0 comments on commit 26a8dad

Please sign in to comment.