def token_type(i):
codepoints = {'(': 'leftParenthesis',
')': 'rightParenthesis',
+ '[': 'leftBracket',
+ ']': 'rightBracket',
+ '{': 'leftBrace',
+ '}': 'rightBrace',
'+': 'plusOrFullStop',
'.': 'plusOrFullStop',
'-': 'hyphenMinus',
+ '*': 'asterisk',
',': 'comma',
'/': 'solidus',
'\\': 'reverseSolidus',
';': 'semiColon',
}
whitespace = '\n\r\t\f '
+ quotes = '"\''
c = chr(i)
if c in whitespace:
return 'whiteSpace'
return 'asciiDigit'
if c.isalpha() or c == '_':
return 'nameStart'
+ if c in quotes:
+ return 'stringStart'
if i == 0:
return 'endOfFile'
return codepoints.get(c)