Lephenixnoir
/
PythonExtra
1
0
Fork 2
Code Issues 9 Pull Requests 3 Projects Releases 2 Wiki Activity

py/lexer: Move check for keyword to name-tokenising block. 

Keywords only needs to be searched for if the token is a MP_TOKEN_NAME, so
we can move the seach to the part of the code that does the tokenising for
MP_TOKEN_NAME.
This commit is contained in:
Damien George 2017-02-17 10:59:57 +11:00
parent 98b3072da5
commit a68c754688
1 changed files with 17 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
py/lexer.c
View File

@ -527,6 +527,23 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
next_char(lex);
}
// Check if the name is a keyword.
// We also check for __debug__ here and convert it to its value. This is
// so the parser gives a syntax error on, eg, x.__debug__. Otherwise, we
// need to check for this special token in many places in the compiler.
// TODO improve speed of these string comparisons
for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) {
if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) {
if (i == MP_ARRAY_SIZE(tok_kw) - 1) {
// tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__"
lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE);
} else {
lex->tok_kind = MP_TOKEN_KW_FALSE + i;
}
break;
}
}
} else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) {
bool forced_integer = false;
if (is_char(lex, '.')) {
@ -655,26 +672,6 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
}
}
}
// check for keywords
if (lex->tok_kind == MP_TOKEN_NAME) {
// We check for __debug__ here and convert it to its value. This is so
// the parser gives a syntax error on, eg, x.__debug__. Otherwise, we
// need to check for this special token in many places in the compiler.
// TODO improve speed of these string comparisons
//for (mp_int_t i = 0; tok_kw[i] != NULL; i++) {
for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) {
if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) {
if (i == MP_ARRAY_SIZE(tok_kw) - 1) {
// tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__"
lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE);
} else {
lex->tok_kind = MP_TOKEN_KW_FALSE + i;
}
break;
}
}
}
}
mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {