scan.ulex

%defs (
  open Token
  type lex_result = Token.token
  val eof = fn () => Token.EOF
  fun unescape s = case (String.fromCString s) of NONE => s | SOME s' => s'
);

%name Lexer;
%states IN_STRING IN_COMMENT;

%let letter = [a-zA-Z\u00c0-\u00ff] & [^\u00d7\u00f7];
%let capital = [A-Z\u00c0-\u00dd] & [^\u00d7];
%let small = [a-z\u00de-\u00ff] & [^\u00f7];
%let digit = [0-9];
%let ident = {letter} | {digit} | [_'];
%let space = [ \t\r\n];
%let quot = ["];
%let backslash = [\\];

<INITIAL> ";" => (Semi);
<INITIAL> "." => (Dot);
<INITIAL> "::=" => (As);
<INITIAL> "[" => (LBracket);
<INITIAL> "]" => (RBracket);
<INITIAL> "_" => (Underscore);
<INITIAL> "(" => (LParen);
<INITIAL> ")" => (RParen);
<INITIAL> ":" => (Colon);
<INITIAL> token => (TokenKw);
<INITIAL> of => (OfKw);
<INITIAL> comment => (CommentKw);
<INITIAL> separator => (SeparatorKw);
<INITIAL> terminator => (TerminatorKw);
<INITIAL> nonempty => (NonemptyKw);
<INITIAL> coercions => (CoercionsKw);
<INITIAL> {letter} {ident}* => (Ident yytext);
<INITIAL> {digit}+ => (Integer (Option.valOf (Int.fromString (yytext))));
<INITIAL> {digit}+ "." {digit}+ ("e" ("-")? {digit}+)? => (Double (Option.valOf (Real.fromString (yytext))));
<INITIAL> {quot}{2} => (String "");
<INITIAL> {quot} => (YYBEGIN IN_STRING; continue ());
<INITIAL> "{-" => (YYBEGIN IN_COMMENT; continue ());
<INITIAL> "--" [^\n]* [\n] => (continue ());
<INITIAL> {space}+ => (continue ());

<IN_STRING> ([^"]|{backslash}{quot})* => (String (unescape yytext)); 
<IN_STRING> {quot} => (YYBEGIN INITIAL; continue ());

<IN_COMMENT> "-}" => (YYBEGIN INITIAL; continue ());
<IN_COMMENT> . => (continue ());