tokens.py

import string
DIGITS = '0123456789'
LETTERS = string.ascii_letters
LETTERS_DIGITS = LETTERS+DIGITS



TT_INT       = 'INT'
TT_FLOAT     = 'FLOAT'
TT_PLUS      = 'PLUS'
TT_MINUS     = 'MINUS'
TT_MUL       = 'MUL'
TT_DIV       = 'DIV'
TT_POW       = 'POW'
TT_LPAREN    = 'LPAREN'
TT_RPAREN    = 'RPAREN'
TT_EOF       = 'EOF'
TT_IDENTIFIER = 'IDENTIFIER'
TT_KEYWORD  =   'KEYWORD'
TT_EQ       =   'EQ' 
TT_NE       =   'NE'
TT_EE       =   'EE'
TT_LT       =   'LT'
TT_GT       =   'GT'
TT_GTE      =   'GTE'
TT_LTE      =   'LTE'
TT_NEWLINE  =   'NEWLINE'


KEYWORDS = [
    'VAR',
    'AND',
    'OR',
    'NOT',
    'IF',
    'THEN',
    'ELIF',
    'ELSE',
    'FOR',
    'TO',
    'STEP',
    'WHILE',
    'END'
]





class Token:
    def __init__(self, type_, value=None, pos_start=None, pos_end=None):
        self.type = type_
        self.value = value

        if pos_start:
            self.pos_start = pos_start.copy()
            self.pos_end = pos_start.copy()
            self.pos_end.advance()

        if pos_end:
            self.pos_end = pos_end.copy()

    def __repr__(self):
        if self.value : return f'{self.type}:{self.value}'
        return f'{self.type}'

    def matches(self, type_, value):
        return self.type==type_ and self.value==value