Lephenixnoir
/
PythonExtra
1
0
Fork 2
Code Issues 9 Pull Requests 3 Projects Releases 2 Wiki Activity

py/parse: Put const bytes objects in parse tree as const object. 

Instead of as an intermediate qstr, which may unnecessarily intern the data
of the bytes object.

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2022-03-16 00:22:58 +11:00
parent 65851ebb51
commit 3c7cab4e98
4 changed files with 12 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
py/compile.c
View File

@ -2816,16 +2816,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
case MP_PARSE_NODE_STRING:
EMIT_ARG(load_const_str, arg);
break;
case MP_PARSE_NODE_BYTES:
// only create and load the actual bytes object on the last pass
if (comp->pass != MP_PASS_EMIT) {
EMIT_ARG(load_const_obj, mp_const_none);
} else {
size_t len;
const byte *data = qstr_data(arg, &len);
EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len));
}
break;
case MP_PARSE_NODE_TOKEN:
default:
if (arg == MP_TOKEN_NEWLINE) {

19
py/parse.c
View File

@ -388,9 +388,6 @@ void mp_parse_node_print(const mp_print_t *print, mp_parse_node_t pn, size_t ind
case MP_PARSE_NODE_STRING:
mp_printf(print, "str(%s)\n", qstr_str(arg));
break;
case MP_PARSE_NODE_BYTES:
mp_printf(print, "bytes(%s)\n", qstr_str(arg));
break;
default:
assert(MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_TOKEN);
mp_printf(print, "tok(%u)\n", (uint)arg);
@ -504,8 +501,8 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) {
} else if (lex->tok_kind == MP_TOKEN_FLOAT_OR_IMAG) {
mp_obj_t o = mp_parse_num_decimal(lex->vstr.buf, lex->vstr.len, true, false, lex);
pn = make_node_const_object(parser, lex->tok_line, o);
} else if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) {
// Don't automatically intern all strings/bytes. doc strings (which are usually large)
} else if (lex->tok_kind == MP_TOKEN_STRING) {
// Don't automatically intern all strings. Doc strings (which are usually large)
// will be discarded by the compiler, and so we shouldn't intern them.
qstr qst = MP_QSTRnull;
if (lex->vstr.len <= MICROPY_ALLOC_PARSE_INTERN_STRING_LEN) {
@ -517,14 +514,16 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) {
}
if (qst != MP_QSTRnull) {
// qstr exists, make a leaf node
pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst);
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_STRING, qst);
} else {
// not interned, make a node holding a pointer to the string/bytes object
mp_obj_t o = mp_obj_new_str_copy(
lex->tok_kind == MP_TOKEN_STRING ? &mp_type_str : &mp_type_bytes,
(const byte *)lex->vstr.buf, lex->vstr.len);
// not interned, make a node holding a pointer to the string object
mp_obj_t o = mp_obj_new_str_copy(&mp_type_str, (const byte *)lex->vstr.buf, lex->vstr.len);
pn = make_node_const_object(parser, lex->tok_line, o);
}
} else if (lex->tok_kind == MP_TOKEN_BYTES) {
// make a node holding a pointer to the bytes object
mp_obj_t o = mp_obj_new_bytes((const byte *)lex->vstr.buf, lex->vstr.len);
pn = make_node_const_object(parser, lex->tok_line, o);
} else {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_TOKEN, lex->tok_kind);
}

6
py/parse.h
View File

@ -39,15 +39,13 @@ struct _mp_lexer_t;
// - xxxx...xx00: pointer to mp_parse_node_struct_t
// - xx...xx0010: an identifier; bits 4 and above are the qstr
// - xx...xx0110: a string; bits 4 and above are the qstr holding the value
// - xx...xx1010: a string of bytes; bits 4 and above are the qstr holding the value
// - xx...xx1110: a token; bits 4 and above are mp_token_kind_t
// - xx...xx1010: a token; bits 4 and above are mp_token_kind_t
#define MP_PARSE_NODE_NULL (0)
#define MP_PARSE_NODE_SMALL_INT (0x1)
#define MP_PARSE_NODE_ID (0x02)
#define MP_PARSE_NODE_STRING (0x06)
#define MP_PARSE_NODE_BYTES (0x0a)
#define MP_PARSE_NODE_TOKEN (0x0e)
#define MP_PARSE_NODE_TOKEN (0x0a)
typedef uintptr_t mp_parse_node_t; // must be pointer size

2
tests/cmdline/cmd_parsetree.py.exp
View File

@ -18,7 +18,7 @@
[ 8] literal \.\+
[ 9] \(rule\|expr_stmt\)(5) (n=2)
id(d)
bytes(bytes)
[ 9] literal \.\+
[ 10] \(rule\|expr_stmt\)(5) (n=2)
id(e)
[ 10] literal \.\+