Skip to content

Commit

Permalink
Merge branch 'refactor/tokenizer' into autobuild/tokenizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@c-lipka
c-lipka committed May 31, 2018
2 parents 9c2ce78 + ac88d0e commit 91c9449
Show file tree
Hide file tree
Showing 9 changed files with 146 additions and 215 deletions.
2 changes: 1 addition & 1 deletion source/base/version.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
/// where `N` is a serial number starting at 1 in each phase, `TIME` is the number of minutes
/// since 2000-01-01 00:00, and `FEATURE` is an arbitrary alphanumeric moniker for a particular
/// experimental feature.
#define POV_RAY_PRERELEASE "x.tokenizer.9675110"
#define POV_RAY_PRERELEASE "x.tokenizer.9684878"

#if defined(DOXYGEN) && !defined(POV_RAY_PRERELEASE)
// Work around doxygen being unable to document undefined macros.
Expand Down
21 changes: 15 additions & 6 deletions source/core/coretypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,18 +610,27 @@ inline void intrusive_ptr_release(GenericFunctionContextFactory* f) { if (!(--f-
typedef intrusive_ptr<GenericFunctionContextFactory> GenericFunctionContextFactoryIPtr;
typedef GenericFunctionContextFactory* GenericFunctionContextFactoryTPtr;

struct SourceInfo : MessageContext
struct SourcePosition
{
UCS2String fileName;
POV_LONG line;
POV_LONG column;
POV_OFF_T offset;
SourcePosition() = default;
SourcePosition(const SourcePosition&) = default;
SourcePosition(POV_LONG l, POV_LONG c, POV_OFF_T o) : line(l), column(c), offset(o) {}
};

struct SourceInfo : MessageContext
{
UCS2String fileName;
SourcePosition position;
SourceInfo() = default;
SourceInfo(const MessageContext& o) : fileName(o.GetFileName()), line(o.GetLine()), column(o.GetColumn()), offset(o.GetOffset()) {}
SourceInfo(const MessageContext& o) : fileName(o.GetFileName()), position(o.GetLine(), o.GetColumn(), o.GetOffset()) {}
SourceInfo(const UCS2String& fn, SourcePosition& p) : fileName(fn), position(p) {}
virtual UCS2String GetFileName() const override { return fileName; }
virtual POV_LONG GetLine() const override { return line; }
virtual POV_LONG GetColumn() const override { return column; }
virtual POV_OFF_T GetOffset() const override { return offset; }
virtual POV_LONG GetLine() const override { return position.line; }
virtual POV_LONG GetColumn() const override { return position.column; }
virtual POV_OFF_T GetOffset() const override { return position.offset; }
};

struct CustomFunctionSourceInfo : SourceInfo
Expand Down
4 changes: 1 addition & 3 deletions source/parser/fncode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,7 @@ FNCode::FNCode(Parser *pa, FunctionCode *f, bool is_local, const char *n)
else
function->sourceInfo.name = POV_STRDUP("");
function->sourceInfo.fileName = parser->UCS2_strdup(parser->mToken.sourceFile->Name());
function->sourceInfo.line = parser->mToken.raw.lexeme.position.line;
function->sourceInfo.column = parser->mToken.raw.lexeme.position.column;
function->sourceInfo.offset = parser->mToken.raw.lexeme.position.offset;
function->sourceInfo.position = parser->mToken.raw.lexeme.position;
function->flags = 0;
function->private_copy_method = nullptr;
function->private_destroy_method = nullptr;
Expand Down
10 changes: 3 additions & 7 deletions source/parser/parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9014,7 +9014,7 @@ bool Parser::Parse_RValue (TokenId Previous, TokenId *NumberPtr, void **DataPtr,
void *Temp_Data;
POV_PARAM *New_Par;
bool Found=true;
int Temp_Count=3000000;
int Temp_Count=3000000; // TODO FIXME - magic value!
bool Old_Ok=Ok_To_Declare;
int Terms;
bool function_identifier;
Expand Down Expand Up @@ -9142,7 +9142,7 @@ bool Parser::Parse_RValue (TokenId Previous, TokenId *NumberPtr, void **DataPtr,
// get the number of tokens found
Temp_Count -= token_count;

// no tokens have been found or a fucntion call had no parameters in parenthesis
// no tokens have been found or a function call had no parameters in parenthesis
if (!((Temp_Count==-1) || (Temp_Count==TOKEN_OVERFLOW_RESET_COUNT)) && had_callable_identifier)
Error("Identifier expected, incomplete function call or spline call found instead.");

Expand Down Expand Up @@ -9457,7 +9457,6 @@ void Parser::Destroy_Ident_Data(void *Data, int Type)
{
int i;
POV_ARRAY *a;
DATA_FILE *Temp_File;

if(Data == nullptr)
return;
Expand Down Expand Up @@ -9563,10 +9562,7 @@ void Parser::Destroy_Ident_Data(void *Data, int Type)
POV_FREE(Data);
break;
case FILE_ID_TOKEN:
Temp_File = reinterpret_cast<DATA_FILE *>(Data);
Temp_File->In_File = nullptr;
Temp_File->Out_File = nullptr;
POV_FREE(Data);
delete reinterpret_cast<DATA_FILE *>(Data);
break;
case FUNCT_ID_TOKEN:
case VECTFUNCT_ID_TOKEN:
Expand Down
37 changes: 28 additions & 9 deletions source/parser/parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,31 @@ class Parser : public SceneTask

struct DATA_FILE
{
shared_ptr<pov_base::ITextStream> In_File;
shared_ptr<RawTokenizer> inTokenizer;
RawToken inToken;
shared_ptr<pov_base::OTextStream> Out_File;
bool inUngetToken : 1;
bool busyParsing : 1; ///< `true` if parsing a statement related to the file, `false` otherwise.
bool R_Flag : 1;

bool ReadNextToken()
{
POV_PARSER_ASSERT(inTokenizer != nullptr);
if (!inUngetToken && !inTokenizer->GetNextToken(inToken))
{
inToken.id = END_OF_FILE_TOKEN;
return false;
}
inUngetToken = false;
return true;
}
void UnReadToken()
{
POV_PARSER_ASSERT(!inUngetToken);
if (inToken.id != END_OF_FILE_TOKEN)
inUngetToken = true;
}

DATA_FILE() : inUngetToken(false), busyParsing(false) {}
};

// constructor
Expand Down Expand Up @@ -409,7 +430,7 @@ class Parser : public SceneTask
void Release_Entry_Reference (SYM_TABLE *table, SYM_ENTRY *Entry);
SYM_ENTRY *Destroy_Entry (SYM_ENTRY *Entry, bool destroyName);
bool Parse_Ifdef_Param ();
int Parse_For_Param (char**, DBL*, DBL*);
int Parse_For_Param (UTF8String&, DBL*, DBL*);

// parstxtr.h/parstxtr.cpp
TEXTURE *Parse_Texture (void);
Expand Down Expand Up @@ -471,7 +492,6 @@ class Parser : public SceneTask
void Parse_Float_Param2 (DBL *Val1, DBL *Val2);
void Init_Random_Generators (void);
void Destroy_Random_Generators (void);
DBL Parse_Signed_Float(void);

// function.h/function.cpp
FUNCTION_PTR Parse_Function(void);
Expand Down Expand Up @@ -586,8 +606,6 @@ class Parser : public SceneTask

int line_count;

bool readingExternalFile;

vector<RawTokenizer::HotBookmark> maIncludeStack;

struct CS_ENTRY
Expand All @@ -598,11 +616,11 @@ class Parser : public SceneTask
bool Macro_Same_Flag;
bool Switch_Case_Ok_Flag;
Macro *PMac;
char* Loop_Identifier;
UTF8String Loop_Identifier;
DBL For_Loop_End;
DBL For_Loop_Step;
CS_ENTRY() : Cond_Type(BUSY_COND), PMac(nullptr), Loop_Identifier(nullptr) {}
~CS_ENTRY() { POV_PARSER_ASSERT(Loop_Identifier == nullptr); }
CS_ENTRY() : Cond_Type(BUSY_COND), PMac(nullptr) {}
~CS_ENTRY() {}
};

vector<CS_ENTRY> Cond_Stack;
Expand Down Expand Up @@ -738,6 +756,7 @@ class Parser : public SceneTask
void Parse_Read(void);
void Parse_Write(void);
int Parse_Read_Value(DATA_FILE *User_File, TokenId Previous, TokenId *NumberPtr, void **DataPtr);
bool Parse_Read_Float_Value(DBL& val, DATA_FILE *User_File);
void Check_Macro_Vers(void);
DBL Parse_Cond_Param(void);
void Parse_Cond_Param2(DBL *V1,DBL *V2);
Expand Down
37 changes: 0 additions & 37 deletions source/parser/parser_expressions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3904,41 +3904,4 @@ void Parser::Destroy_Random_Generators()
Number_Of_Random_Generators = 0;
}

DBL Parser::Parse_Signed_Float(void)
{
DBL Sign=1.0;
DBL Val=0.0;
bool old_allow_id = Allow_Identifier_In_Call;
Allow_Identifier_In_Call = false;

EXPECT
CASE (PLUS_TOKEN)
END_CASE

CASE (DASH_TOKEN)
Sign=-1.0;
Get_Token();
// FALLTHROUGH
CASE (FLOAT_FUNCT_TOKEN)
if (mToken.Function_Id==FLOAT_TOKEN)
{
Val = Sign * mToken.Token_Float;
EXIT
}
else
{
Parse_Error(FLOAT_TOKEN);
}
END_CASE

OTHERWISE
Parse_Error(FLOAT_TOKEN);
END_CASE
END_EXPECT

Allow_Identifier_In_Call = old_allow_id;

return(Val);
}

}
Loading

0 comments on commit 91c9449

Please sign in to comment.