py/persistentcode: Remove remaining native qstr linking support.

Support for architecture-specific qstr linking was removed in
d4d53e9e11, where native code was changed to
access qstr values via qstr_table.  The only remaining use for the special
qstr link table in persistentcode.c is to support native module written in
C, linked via mpy_ld.py.  But native modules can also use the standard
module-level qstr_table (and obj_table) which was introduced in the .mpy
file reworking in f2040bfc7e.

This commit removes the remaining native qstr liking support in
persistentcode.c's load_raw_code function, and adds two new relocation
options for constants.qstr_table and constants.obj_table.  mpy_ld.py is
updated to use these relocations options instead of the native qstr link
table.

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2022-05-26 10:51:29 +10:00
parent 2111ca0b8f
commit b37b578214
5 changed files with 56 additions and 96 deletions

View File

@ -71,6 +71,7 @@ STATIC size_t read_uint(mp_reader_t *reader);
typedef struct _reloc_info_t {
mp_reader_t *reader;
mp_module_context_t *context;
uint8_t *rodata;
uint8_t *bss;
} reloc_info_t;
@ -112,11 +113,17 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) {
dest = (uintptr_t)ri->bss;
}
} else if (op == 6) {
// Destination is qstr_table
dest = (uintptr_t)ri->context->constants.qstr_table;
} else if (op == 7) {
// Destination is obj_table
dest = (uintptr_t)ri->context->constants.obj_table;
} else if (op == 8) {
// Destination is mp_fun_table itself
dest = (uintptr_t)&mp_fun_table;
} else {
// Destination is an entry in mp_fun_table
dest = ((uintptr_t *)&mp_fun_table)[op - 7];
dest = ((uintptr_t *)&mp_fun_table)[op - 9];
}
while (n--) {
*addr_to_adjust++ += dest;
@ -205,7 +212,7 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
}
}
STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *context) {
// Load function kind and data length
size_t kind_len = read_uint(reader);
int kind = (kind_len & 3) + MP_CODE_BYTECODE;
@ -239,24 +246,6 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
MP_PLAT_ALLOC_EXEC(fun_data_len, (void **)&fun_data, &fun_alloc);
read_bytes(reader, fun_data, fun_data_len);
if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) {
// Parse qstr link table and link native code
size_t n_qstr_link = read_uint(reader);
for (size_t i = 0; i < n_qstr_link; ++i) {
size_t off = read_uint(reader);
qstr qst = load_qstr(reader);
uint8_t *dest = fun_data + (off >> 2);
if ((off & 3) == 0) {
// Generic 16-bit link
dest[0] = qst & 0xff;
dest[1] = (qst >> 8) & 0xff;
} else if ((off & 3) == 3) {
// Generic, aligned qstr-object link
*(mp_obj_t *)dest = MP_OBJ_NEW_QSTR(qst);
}
}
}
if (kind == MP_CODE_NATIVE_PY) {
// Read prelude offset within fun_data, and extract scope flags.
prelude_offset = read_uint(reader);
@ -315,7 +304,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
n_children = read_uint(reader);
children = m_new(mp_raw_code_t *, n_children + (kind == MP_CODE_NATIVE_PY));
for (size_t i = 0; i < n_children; ++i) {
children[i] = load_raw_code(reader);
children[i] = load_raw_code(reader, context);
}
}
@ -349,7 +338,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
#endif
// Relocate and commit code to executable address space
reloc_info_t ri = {reader, rodata, bss};
reloc_info_t ri = {reader, context, rodata, bss};
#if defined(MP_PLAT_COMMIT_EXEC)
void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL;
fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri);
@ -429,7 +418,7 @@ mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *
// Load top-level module.
mp_compiled_module_t cm2;
cm2.rc = load_raw_code(reader);
cm2.rc = load_raw_code(reader, context);
cm2.context = context;
#if MICROPY_PERSISTENT_CODE_SAVE
@ -567,11 +556,6 @@ STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) {
mp_print_bytes(print, rc->fun_data, rc->fun_data_len);
#if MICROPY_EMIT_MACHINE_CODE
if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) {
// Save qstr link table for native code
mp_print_uint(print, 0);
}
if (rc->kind == MP_CODE_NATIVE_PY) {
// Save prelude size
mp_print_uint(print, rc->prelude_offset);

View File

@ -74,9 +74,7 @@ user_files = {
b'\x02' # 2 children
b'\x42' # 8 bytes, no children, viper code
b'\x00\x00\x00\x00\x00\x00' # dummy machine code
b'\x00\x00' # slot for qstr0
b'\x01\x0c\x0aprint\x00' # n_qstr=1, qstr0
b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code
b'\x00' # scope_flags
b'\x43' # 8 bytes, no children, asm code
@ -102,7 +100,6 @@ user_files = {
b'\x22' # 4 bytes, no children, viper code
b'\x00\x00\x00\x00' # dummy machine code
b'\x00' # n_qstr=0
b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC
b'\x06\x04' # rodata=6 bytes, bss=4 bytes
b'rodata' # rodata content

View File

@ -49,9 +49,9 @@ class UserFS:
# by the required value of sys.implementation._mpy.
features0_file_contents = {
# -march=x64
0x806: b'M\x06\b\x1f\x01\x004build/features0.native.mpy\x00\x8aB\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x85\x00\x12factorial\x00\x10\r$\x01&\x9f \x01"\xff',
0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
# -march=armv6m
0x1006: b'M\x06\x10\x1f\x01\x004build/features0.native.mpy\x00\x88"\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyD\x18\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial\x00\x10\r<\x01>\x9f8\x01:\xff',
0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
}
# Populate armv7m-derived archs based on armv6m.

View File

@ -976,7 +976,6 @@ class RawCodeNative(RawCode):
kind,
fun_data,
prelude_offset,
qstr_links,
scope_flags,
n_pos_args,
type_sig,
@ -989,7 +988,6 @@ class RawCodeNative(RawCode):
self.scope_flags = scope_flags
self.n_pos_args = n_pos_args
self.qstr_links = qstr_links
self.type_sig = type_sig
if config.native_arch in (
MP_NATIVE_ARCH_X86,
@ -1196,15 +1194,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments):
rc = RawCodeBytecode(cm_escaped_name, qstr_table, obj_table, fun_data)
else:
# Create native raw code.
qstr_links = []
if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER):
# Read qstr link table.
n_qstr_link = reader.read_uint()
for _ in range(n_qstr_link):
off = reader.read_uint()
qst = read_qstr(reader, segments)
qstr_links.append((off >> 2, off & 3, qst))
native_scope_flags = 0
native_n_pos_args = 0
native_type_sig = 0
@ -1242,7 +1231,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments):
kind,
fun_data,
prelude_offset,
qstr_links,
native_scope_flags,
native_n_pos_args,
native_type_sig,

View File

@ -36,6 +36,8 @@ import makeqstrdata as qstrutil
# MicroPython constants
MPY_VERSION = 6
MP_CODE_BYTECODE = 2
MP_CODE_NATIVE_VIPER = 4
MP_NATIVE_ARCH_X86 = 1
MP_NATIVE_ARCH_X64 = 2
MP_NATIVE_ARCH_ARMV6M = 4
@ -44,8 +46,7 @@ MP_NATIVE_ARCH_ARMV7EMSP = 7
MP_NATIVE_ARCH_ARMV7EMDP = 8
MP_NATIVE_ARCH_XTENSA = 9
MP_NATIVE_ARCH_XTENSAWIN = 10
MP_CODE_BYTECODE = 2
MP_CODE_NATIVE_VIPER = 4
MP_PERSISTENT_OBJ_STR = 5
MP_SCOPE_FLAG_VIPERRELOC = 0x10
MP_SCOPE_FLAG_VIPERRODATA = 0x20
MP_SCOPE_FLAG_VIPERBSS = 0x40
@ -110,21 +111,20 @@ def asm_jump_xtensa(entry):
class ArchData:
def __init__(self, name, mpy_feature, qstr_entry_size, word_size, arch_got, asm_jump):
def __init__(self, name, mpy_feature, word_size, arch_got, asm_jump, *, separate_rodata=False):
self.name = name
self.mpy_feature = mpy_feature
self.qstr_entry_size = qstr_entry_size
self.qstr_entry_size = 2
self.word_size = word_size
self.arch_got = arch_got
self.asm_jump = asm_jump
self.separate_rodata = name == "EM_XTENSA" and qstr_entry_size == 4
self.separate_rodata = separate_rodata
ARCH_DATA = {
"x86": ArchData(
"EM_386",
MP_NATIVE_ARCH_X86 << 2,
2,
4,
(R_386_PC32, R_386_GOT32, R_386_GOT32X),
asm_jump_x86,
@ -132,7 +132,6 @@ ARCH_DATA = {
"x64": ArchData(
"EM_X86_64",
MP_NATIVE_ARCH_X64 << 2,
2,
8,
(R_X86_64_GOTPCREL, R_X86_64_REX_GOTPCRELX),
asm_jump_x86,
@ -140,7 +139,6 @@ ARCH_DATA = {
"armv6m": ArchData(
"EM_ARM",
MP_NATIVE_ARCH_ARMV6M << 2,
2,
4,
(R_ARM_GOT_BREL,),
asm_jump_thumb,
@ -148,7 +146,6 @@ ARCH_DATA = {
"armv7m": ArchData(
"EM_ARM",
MP_NATIVE_ARCH_ARMV7M << 2,
2,
4,
(R_ARM_GOT_BREL,),
asm_jump_thumb2,
@ -156,7 +153,6 @@ ARCH_DATA = {
"armv7emsp": ArchData(
"EM_ARM",
MP_NATIVE_ARCH_ARMV7EMSP << 2,
2,
4,
(R_ARM_GOT_BREL,),
asm_jump_thumb2,
@ -164,7 +160,6 @@ ARCH_DATA = {
"armv7emdp": ArchData(
"EM_ARM",
MP_NATIVE_ARCH_ARMV7EMDP << 2,
2,
4,
(R_ARM_GOT_BREL,),
asm_jump_thumb2,
@ -172,7 +167,6 @@ ARCH_DATA = {
"xtensa": ArchData(
"EM_XTENSA",
MP_NATIVE_ARCH_XTENSA << 2,
2,
4,
(R_XTENSA_32, R_XTENSA_PLT),
asm_jump_xtensa,
@ -181,9 +175,9 @@ ARCH_DATA = {
"EM_XTENSA",
MP_NATIVE_ARCH_XTENSAWIN << 2,
4,
4,
(R_XTENSA_32, R_XTENSA_PLT),
asm_jump_xtensa,
separate_rodata=True,
),
}
@ -435,8 +429,8 @@ def populate_got(env):
# Create a relocation for each GOT entry
for got_entry in got_list:
if got_entry.name == "mp_fun_table":
dest = "mp_fun_table"
if got_entry.name in ("mp_native_qstr_table", "mp_native_obj_table", "mp_fun_table"):
dest = got_entry.name
elif got_entry.name.startswith("mp_fun_table+0x"):
dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size
elif got_entry.sec_name.startswith(".text"):
@ -754,19 +748,19 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len):
env.lit_section = Section("LIT", bytearray(lit_size), env.arch.word_size)
env.sections.insert(1, env.lit_section)
# Create section to contain mp_native_qstr_val_table
env.qstr_val_section = Section(
".text.QSTR_VAL",
# Create section to contain mp_native_qstr_table
env.qstr_table_section = Section(
".external.qstr_table",
bytearray(native_qstr_vals_len * env.arch.qstr_entry_size),
env.arch.qstr_entry_size,
)
env.sections.append(env.qstr_val_section)
# Create section to contain mp_native_qstr_obj_table
env.qstr_obj_section = Section(
".text.QSTR_OBJ", bytearray(native_qstr_objs_len * env.arch.word_size), env.arch.word_size
# Create section to contain mp_native_obj_table
env.obj_table_section = Section(
".external.obj_table",
bytearray(native_qstr_objs_len * env.arch.word_size),
env.arch.word_size,
)
env.sections.append(env.qstr_obj_section)
# Resolve unknown symbols
mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0)
@ -796,10 +790,10 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len):
pass
elif sym.name == "mp_fun_table":
sym.section = Section(".external", b"", 0)
elif sym.name == "mp_native_qstr_val_table":
sym.section = env.qstr_val_section
elif sym.name == "mp_native_qstr_obj_table":
sym.section = env.qstr_obj_section
elif sym.name == "mp_native_qstr_table":
sym.section = env.qstr_table_section
elif sym.name == "mp_native_obj_table":
sym.section = env.obj_table_section
elif sym.name in env.known_syms:
sym.resolved = env.known_syms[sym.name]
else:
@ -923,13 +917,21 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
# MPY: n_qstr
out.write_uint(1)
out.write_uint(1 + len(native_qstr_vals))
# MPY: n_obj
out.write_uint(0)
out.write_uint(len(native_qstr_objs))
# MPY: qstr table
out.write_qstr(fmpy) # filename
for q in native_qstr_vals:
out.write_qstr(q)
# MPY: object table
for q in native_qstr_objs:
out.write_bytes(bytearray([MP_PERSISTENT_OBJ_STR]))
out.write_uint(len(q))
out.write_bytes(bytes(q, "utf8") + b"\x00")
# MPY: kind/len
out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE))
@ -937,17 +939,6 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
# MPY: machine code
out.write_bytes(env.full_text)
# MPY: n_qstr_link (assumes little endian)
out.write_uint(len(native_qstr_vals) + len(native_qstr_objs))
for q in range(len(native_qstr_vals)):
off = env.qstr_val_section.addr + q * env.arch.qstr_entry_size
out.write_uint(off << 2)
out.write_qstr(native_qstr_vals[q])
for q in range(len(native_qstr_objs)):
off = env.qstr_obj_section.addr + q * env.arch.word_size
out.write_uint(off << 2 | 3)
out.write_qstr(native_qstr_objs[q])
# MPY: scope_flags
scope_flags = MP_SCOPE_FLAG_VIPERRELOC
if len(env.full_rodata):
@ -978,10 +969,14 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
kind = 0
elif isinstance(kind, str) and kind.startswith(".bss"):
kind = bss_const_table_idx
elif kind == "mp_fun_table":
elif kind == "mp_native_qstr_table":
kind = 6
elif kind == "mp_native_obj_table":
kind = 7
elif kind == "mp_fun_table":
kind = 8
else:
kind = 7 + kind
kind = 9 + kind
assert addr % env.arch.word_size == 0, addr
offset = addr // env.arch.word_size
if kind == prev_kind and base == prev_base and offset == prev_offset + 1:
@ -1023,18 +1018,14 @@ def do_preprocess(args):
for i, q in enumerate(static_qstrs):
print("#define %s (%u)" % (q, i + 1), file=f)
for i, q in enumerate(sorted(qstr_vals)):
print("#define %s (mp_native_qstr_val_table[%d])" % (q, i), file=f)
print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f)
for i, q in enumerate(sorted(qstr_objs)):
print(
"#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_qstr_obj_table[%d])" % (q, i),
"#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_obj_table[%d])" % (q, i),
file=f,
)
if args.arch == "xtensawin":
qstr_type = "uint32_t" # esp32 can only read 32-bit values from IRAM
else:
qstr_type = "uint16_t"
print("extern const {} mp_native_qstr_val_table[];".format(qstr_type), file=f)
print("extern const mp_uint_t mp_native_qstr_obj_table[];", file=f)
print("extern const uint16_t mp_native_qstr_table[];", file=f)
print("extern const mp_uint_t mp_native_obj_table[];", file=f)
def do_link(args):