mibi88
/
PythonExtra
forked from Lephenixnoir/PythonExtra
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

py/bc: Provide separate code-state setup funcs for bytecode and native. 

mpy-cross will now generate native code based on the size of
mp_code_state_native_t, and the runtime will use this struct to calculate
the offset of the .state field.  This makes native code generation and
execution (which rely on this struct) independent to the settings
MICROPY_STACKLESS and MICROPY_PY_SYS_SETTRACE, both of which change the
size of the mp_code_state_t struct.

Fixes issue #5059.

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2022-05-15 13:37:06 +10:00
parent 8e1db993cd
commit 1762990579
7 changed files with 98 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
py/bc.c
View File

@ -122,23 +122,15 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) {
// contain the following valid entries:
// - code_state->fun_bc should contain a pointer to the function object
// - code_state->ip should contain a pointer to the beginning of the prelude
// - code_state->sp should be: &code_state->state[0] - 1
// - code_state->n_state should be the number of objects in the local state
void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) {
STATIC void mp_setup_code_state_helper(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) {
// This function is pretty complicated. It's main aim is to be efficient in speed and RAM
// usage for the common case of positional only args.
// get the function object that we want to set up (could be bytecode or native code)
mp_obj_fun_bc_t *self = code_state->fun_bc;
#if MICROPY_STACKLESS
code_state->prev = NULL;
#endif
#if MICROPY_PY_SYS_SETTRACE
code_state->prev_state = NULL;
code_state->frame = NULL;
#endif
// Get cached n_state (rather than decode it again)
size_t n_state = code_state->n_state;
@ -149,16 +141,16 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
(void)n_state_unused;
(void)n_exc_stack_unused;
code_state->sp = &code_state->state[0] - 1;
mp_obj_t *code_state_state = code_state->sp + 1;
code_state->exc_sp_idx = 0;
// zero out the local stack to begin with
memset(code_state->state, 0, n_state * sizeof(*code_state->state));
memset(code_state_state, 0, n_state * sizeof(*code_state->state));
const mp_obj_t *kwargs = args + n_args;
// var_pos_kw_args points to the stack where the var-args tuple, and var-kw dict, should go (if they are needed)
mp_obj_t *var_pos_kw_args = &code_state->state[n_state - 1 - n_pos_args - n_kwonly_args];
mp_obj_t *var_pos_kw_args = &code_state_state[n_state - 1 - n_pos_args - n_kwonly_args];
// check positional arguments
@ -181,7 +173,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
if (n_args >= (size_t)(n_pos_args - n_def_pos_args)) {
// given enough arguments, but may need to use some default arguments
for (size_t i = n_args; i < n_pos_args; i++) {
code_state->state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)];
code_state_state[n_state - 1 - i] = self->extra_args[i - (n_pos_args - n_def_pos_args)];
}
} else {
fun_pos_args_mismatch(self, n_pos_args - n_def_pos_args, n_args);
@ -191,14 +183,14 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
// copy positional args into state
for (size_t i = 0; i < n_args; i++) {
code_state->state[n_state - 1 - i] = args[i];
code_state_state[n_state - 1 - i] = args[i];
}
// check keyword arguments
if (n_kw != 0 || (scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) {
DEBUG_printf("Initial args: ");
dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
mp_obj_t dict = MP_OBJ_NULL;
if ((scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) {
@ -220,11 +212,11 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
arg_qstr = self->context->constants.qstr_table[arg_qstr];
#endif
if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) {
if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) {
if (code_state_state[n_state - 1 - j] != MP_OBJ_NULL) {
mp_raise_msg_varg(&mp_type_TypeError,
MP_ERROR_TEXT("function got multiple values for argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name));
}
code_state->state[n_state - 1 - j] = kwargs[2 * i + 1];
code_state_state[n_state - 1 - j] = kwargs[2 * i + 1];
goto continue2;
}
}
@ -242,10 +234,10 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
}
DEBUG_printf("Args with kws flattened: ");
dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
// fill in defaults for positional args
mp_obj_t *d = &code_state->state[n_state - n_pos_args];
mp_obj_t *d = &code_state_state[n_state - n_pos_args];
mp_obj_t *s = &self->extra_args[n_def_pos_args - 1];
for (size_t i = n_def_pos_args; i > 0; i--, d++, s--) {
if (*d == MP_OBJ_NULL) {
@ -254,13 +246,13 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
}
DEBUG_printf("Args after filling default positional: ");
dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
// Check that all mandatory positional args are specified
while (d < &code_state->state[n_state]) {
while (d < &code_state_state[n_state]) {
if (*d++ == MP_OBJ_NULL) {
mp_raise_msg_varg(&mp_type_TypeError,
MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state->state[n_state] - d);
MP_ERROR_TEXT("function missing required positional argument #%d"), &code_state_state[n_state] - d);
}
}
@ -275,13 +267,13 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE
arg_qstr = self->context->constants.qstr_table[arg_qstr];
#endif
if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) {
if (code_state_state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) {
mp_map_elem_t *elem = NULL;
if ((scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) {
elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, MP_OBJ_NEW_QSTR(arg_qstr), MP_MAP_LOOKUP);
}
if (elem != NULL) {
code_state->state[n_state - 1 - n_pos_args - i] = elem->value;
code_state_state[n_state - 1 - n_pos_args - i] = elem->value;
} else {
mp_raise_msg_varg(&mp_type_TypeError,
MP_ERROR_TEXT("function missing required keyword argument '%q'"), arg_qstr);
@ -305,18 +297,47 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
// bytecode prelude: initialise closed over variables
for (; n_cell; --n_cell) {
size_t local_num = *ip++;
code_state->state[n_state - 1 - local_num] =
mp_obj_new_cell(code_state->state[n_state - 1 - local_num]);
code_state_state[n_state - 1 - local_num] =
mp_obj_new_cell(code_state_state[n_state - 1 - local_num]);
}
// now that we skipped over the prelude, set the ip for the VM
code_state->ip = ip;
DEBUG_printf("Calling: n_pos_args=%d, n_kwonly_args=%d\n", n_pos_args, n_kwonly_args);
dump_args(code_state->state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
dump_args(code_state->state, n_state);
dump_args(code_state_state + n_state - n_pos_args - n_kwonly_args, n_pos_args + n_kwonly_args);
dump_args(code_state_state, n_state);
}
// On entry code_state should be allocated somewhere (stack/heap) and
// contain the following valid entries:
// - code_state->fun_bc should contain a pointer to the function object
// - code_state->n_state should be the number of objects in the local state
void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) {
code_state->ip = code_state->fun_bc->bytecode;
code_state->sp = &code_state->state[0] - 1;
#if MICROPY_STACKLESS
code_state->prev = NULL;
#endif
#if MICROPY_PY_SYS_SETTRACE
code_state->prev_state = NULL;
code_state->frame = NULL;
#endif
mp_setup_code_state_helper(code_state, n_args, n_kw, args);
}
#if MICROPY_EMIT_NATIVE
// On entry code_state should be allocated somewhere (stack/heap) and
// contain the following valid entries:
// - code_state->fun_bc should contain a pointer to the function object
// - code_state->ip should contain a pointer to the beginning of the prelude
// - code_state->n_state should be the number of objects in the local state
void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) {
code_state->sp = &code_state->state[0] - 1;
mp_setup_code_state_helper((mp_code_state_t *)code_state, n_args, n_kw, args);
}
#endif
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
// The following table encodes the number of bytes that a specific opcode

12
py/bc.h
View File

@ -254,6 +254,17 @@ typedef struct _mp_code_state_t {
// mp_exc_stack_t exc_state[0];
} mp_code_state_t;
// State for an executing native function (based on mp_code_state_t).
typedef struct _mp_code_state_native_t {
struct _mp_obj_fun_bc_t *fun_bc;
const byte *ip;
mp_obj_t *sp;
uint16_t n_state;
uint16_t exc_sp_idx;
mp_obj_dict_t *old_globals;
mp_obj_t state[0];
} mp_code_state_native_t;
// Allocator may return NULL, in which case data is not stored (can be used to compute size).
typedef uint8_t *(*mp_encode_uint_allocator_t)(void *env, size_t nbytes);
@ -269,6 +280,7 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state,
mp_obj_t inject_exc);
mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args);
void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args);
void mp_setup_code_state_native(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args);
void mp_bytecode_print(const mp_print_t *print, const struct _mp_raw_code_t *rc, const mp_module_constants_t *cm);
void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm);
const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, const byte *ip, struct _mp_raw_code_t *const *child_table, const mp_module_constants_t *cm);

18
py/emitnative.c
View File

@ -63,7 +63,7 @@
// C stack layout for native functions:
// 0: nlr_buf_t [optional]
// emit->code_state_start: mp_code_state_t
// emit->code_state_start: mp_code_state_native_t
// emit->stack_start: Python object stack | emit->n_state
// locals (reversed, L0 at end) |
//
@ -71,7 +71,7 @@
// 0=emit->stack_start: nlr_buf_t
//
// Then REG_GENERATOR_STATE points to:
// 0=emit->code_state_start: mp_code_state_t
// 0=emit->code_state_start: mp_code_state_native_t
// emit->stack_start: Python object stack | emit->n_state
// locals (reversed, L0 at end) |
//
@ -88,12 +88,12 @@
#else
#define SIZEOF_NLR_BUF (sizeof(nlr_buf_t) / sizeof(uintptr_t))
#endif
#define SIZEOF_CODE_STATE (sizeof(mp_code_state_t) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_t, state) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t))
#define SIZEOF_CODE_STATE (sizeof(mp_code_state_native_t) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_native_t, state) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_native_t, fun_bc) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_native_t, ip) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_native_t, sp) / sizeof(uintptr_t))
#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_native_t, n_state) / sizeof(uintptr_t))
#define OFFSETOF_OBJ_FUN_BC_CONTEXT (offsetof(mp_obj_fun_bc_t, context) / sizeof(uintptr_t))
#define OFFSETOF_OBJ_FUN_BC_CHILD_TABLE (offsetof(mp_obj_fun_bc_t, child_table) / sizeof(uintptr_t))
#define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t))
@ -412,7 +412,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
// generate code for entry to function
// Work out start of code state (mp_code_state_t or reduced version for viper)
// Work out start of code state (mp_code_state_native_t or reduced version for viper)
emit->code_state_start = 0;
if (NEED_GLOBAL_EXC_HANDLER(emit)) {
emit->code_state_start = SIZEOF_NLR_BUF;

2
py/nativeglue.c
View File

@ -302,7 +302,7 @@ const mp_fun_table_t mp_fun_table = {
mp_delete_global,
mp_obj_new_closure,
mp_arg_check_num_sig,
mp_setup_code_state,
mp_setup_code_state_native,
mp_small_int_floor_divide,
mp_small_int_modulo,
mp_native_yield_from,

2
py/nativeglue.h
View File

@ -131,7 +131,7 @@ typedef struct _mp_fun_table_t {
void (*delete_global)(qstr qst);
mp_obj_t (*new_closure)(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed);
void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig);
void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args);
void (*setup_code_state_native)(mp_code_state_native_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args);
mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom);
mp_int_t (*small_int_modulo)(mp_int_t dividend, mp_int_t divisor);
bool (*yield_from)(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *ret_value);

1
py/objfun.c
View File

@ -209,7 +209,6 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) {
#define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \
code_state->fun_bc = _fun_bc; \
code_state->ip = _fun_bc->bytecode; \
code_state->n_state = _n_state; \
mp_setup_code_state(code_state, n_args, n_kw, args); \
code_state->old_globals = mp_globals_get();

30
py/objgenerator.c
View File

@ -65,7 +65,6 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons
o->pend_exc = mp_const_none;
o->code_state.fun_bc = self_fun;
o->code_state.ip = self_fun->bytecode;
o->code_state.n_state = n_state;
mp_setup_code_state(&o->code_state, n_args, n_kw, args);
return MP_OBJ_FROM_PTR(o);
@ -87,6 +86,13 @@ const mp_obj_type_t mp_type_gen_wrap = {
#if MICROPY_EMIT_NATIVE
// Based on mp_obj_gen_instance_t.
typedef struct _mp_obj_gen_instance_native_t {
mp_obj_base_t base;
mp_obj_t pend_exc;
mp_code_state_native_t code_state;
} mp_obj_gen_instance_native_t;
STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
// The state for a native generating function is held in the same struct as a bytecode function
mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in);
@ -106,14 +112,15 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k
MP_BC_PRELUDE_SIG_DECODE(ip);
// Allocate the generator object, with room for local stack (exception stack not needed).
mp_obj_gen_instance_t *o = mp_obj_malloc_var(mp_obj_gen_instance_t, byte, n_state * sizeof(mp_obj_t), &mp_type_gen_instance);
mp_obj_gen_instance_native_t *o = mp_obj_malloc_var(mp_obj_gen_instance_native_t, byte, n_state * sizeof(mp_obj_t), &mp_type_gen_instance);
// Parse the input arguments and set up the code state
o->pend_exc = mp_const_none;
o->code_state.fun_bc = self_fun;
o->code_state.ip = prelude_ptr;
o->code_state.n_state = n_state;
mp_setup_code_state(&o->code_state, n_args, n_kw, args);
o->code_state.sp = &o->code_state.state[0] - 1;
mp_setup_code_state_native(&o->code_state, n_args, n_kw, args);
// Indicate we are a native function, which doesn't use this variable
o->code_state.exc_sp_idx = MP_CODE_STATE_EXC_SP_IDX_SENTINEL;
@ -171,7 +178,13 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
#endif
// If the generator is started, allow sending a value.
if (self->code_state.sp == self->code_state.state - 1) {
void *state_start = self->code_state.state - 1;
#if MICROPY_EMIT_NATIVE
if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
state_start = ((mp_obj_gen_instance_native_t *)self)->code_state.state - 1;
}
#endif
if (self->code_state.sp == state_start) {
if (send_value != mp_const_none) {
mp_raise_TypeError(MP_ERROR_TEXT("can't send non-None value to a just-started generator"));
}
@ -226,7 +239,14 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
case MP_VM_RETURN_EXCEPTION: {
self->code_state.ip = 0;
*ret_val = self->code_state.state[0];
#if MICROPY_EMIT_NATIVE
if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
*ret_val = ((mp_obj_gen_instance_native_t *)self)->code_state.state[0];
} else
#endif
{
*ret_val = self->code_state.state[0];
}
// PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError
if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(*ret_val)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
*ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator raised StopIteration"));