diff --git a/ports/cc3200/bootmgr/bootloader.mk b/ports/cc3200/bootmgr/bootloader.mk index e5b817391..3aeda062c 100644 --- a/ports/cc3200/bootmgr/bootloader.mk +++ b/ports/cc3200/bootmgr/bootloader.mk @@ -134,3 +134,7 @@ $(HEADER_BUILD)/mpversion.h: | $(HEADER_BUILD) # Create an empty "moduledefs.h" needed by py/mkrules.mk $(HEADER_BUILD)/moduledefs.h: | $(HEADER_BUILD) touch $@ + +# Create an empty "root_pointers.h" needed by py/mkrules.mk +$(HEADER_BUILD)/root_pointers.h: | $(HEADER_BUILD) + touch $@ diff --git a/ports/stm32/mboot/Makefile b/ports/stm32/mboot/Makefile index db39daf88..4c903b4ff 100755 --- a/ports/stm32/mboot/Makefile +++ b/ports/stm32/mboot/Makefile @@ -213,6 +213,7 @@ PREFIX_FILE = ../boards/stm32f4xx_prefix.c BOARD_PINS = $(BOARD_DIR)/pins.csv HEADER_BUILD = $(BUILD)/genhdr GEN_QSTRDEFS_GENERATED = $(HEADER_BUILD)/qstrdefs.generated.h +GEN_ROOT_POINTERS = $(HEADER_BUILD)/root_pointers.h GEN_PINS_SRC = $(BUILD)/pins_$(BOARD).c GEN_PINS_HDR = $(HEADER_BUILD)/pins.h GEN_PINS_QSTR = $(HEADER_BUILD)/pins_qstr.h @@ -220,7 +221,7 @@ GEN_PINS_AF_CONST = $(HEADER_BUILD)/pins_af_const.h GEN_PINS_AF_DEFS = $(HEADER_BUILD)/pins_af_defs.h GEN_PINS_AF_PY = $(BUILD)/pins_af.py -$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_PINS_AF_DEFS) +$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_ROOT_POINTERS) $(GEN_PINS_AF_DEFS) $(HEADER_BUILD): $(MKDIR) -p $(BUILD)/genhdr @@ -228,6 +229,9 @@ $(HEADER_BUILD): $(GEN_QSTRDEFS_GENERATED): | $(HEADER_BUILD) $(Q)echo "// empty" > $@ +$(GEN_ROOT_POINTERS): | $(HEADER_BUILD) + $(Q)echo "// empty" > $@ + $(GEN_PINS_AF_DEFS): $(BOARD_PINS) $(MAKE_PINS) ../$(AF_FILE) $(PREFIX_FILE) | $(HEADER_BUILD) $(ECHO) "GEN $@" $(Q)$(PYTHON) $(MAKE_PINS) --board $(BOARD_PINS) --af ../$(AF_FILE) \ diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets index 84520078f..11c5eae8a 100644 --- a/ports/windows/msvc/genhdr.targets +++ b/ports/windows/msvc/genhdr.targets @@ -4,7 +4,7 @@ - + @@ -15,6 +15,7 @@ $(DestDir)qstrdefscollected.h $(DestDir)qstrdefs.generated.h $(DestDir)/moduledefs.collected + $(DestDir)/root_pointers.collected $(MICROPY_CPYTHON3) python cl.exe @@ -45,7 +46,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { foreach(var inFile in InputFiles) foreach(var line in System.IO.File.ReadAllLines(inFile)) - if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR")) + if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR") || line.Contains("MP_REGISTER")) outFile.WriteLine( line ); } ]]> @@ -114,6 +115,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { + + + + + + + + $(DestDir)root_pointers.h + $(DestFile).tmp + + + + + $(QstrGen).tmp @@ -132,7 +147,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { - + diff --git a/py/make_root_pointers.py b/py/make_root_pointers.py new file mode 100644 index 000000000..efe398b82 --- /dev/null +++ b/py/make_root_pointers.py @@ -0,0 +1,55 @@ +""" +This pre-processor parses a single file containing a list of +MP_REGISTER_ROOT_POINTER(variable declaration) items. + +These are used to generate a header with the required entries for +"struct _mp_state_vm_t" in py/mpstate.h +""" + +from __future__ import print_function + +import argparse +import io +import re + + +PATTERN = re.compile(r"MP_REGISTER_ROOT_POINTER\((.*?)\);") + + +def find_root_pointer_registrations(filename): + """Find any MP_REGISTER_ROOT_POINTER definitions in the provided file. + + :param str filename: path to file to check + :return: List[variable_declaration] + """ + with io.open(filename, encoding="utf-8") as c_file_obj: + return set(re.findall(PATTERN, c_file_obj.read())) + + +def generate_root_pointer_header(root_pointers): + """Generate header with root pointer entries. + + :param List[variable_declaration] root_pointers: root pointer declarations + :return: None + """ + + # Print header file for all external modules. + print("// Automatically generated by make_root_pointers.py.") + print() + + for item in root_pointers: + print(item, end=";") + print() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("file", nargs=1, help="file with MP_REGISTER_ROOT_POINTER definitions") + args = parser.parse_args() + + root_pointers = find_root_pointer_registrations(args.file[0]) + generate_root_pointer_header(sorted(root_pointers)) + + +if __name__ == "__main__": + main() diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 4c416a874..c445d6d1f 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -24,6 +24,9 @@ _MODE_COMPRESS = "compress" # Extract MP_REGISTER_MODULE(...) macros. _MODE_MODULE = "module" +# Extract MP_REGISTER_ROOT_POINTER(...) macros. +_MODE_ROOT_POINTER = "root_pointer" + def is_c_source(fname): return os.path.splitext(fname)[1] in [".c"] @@ -90,6 +93,8 @@ def process_file(f): re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)') elif args.mode == _MODE_MODULE: re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);") + elif args.mode == _MODE_ROOT_POINTER: + re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);") output = [] last_fname = None for line in f: @@ -111,7 +116,7 @@ def process_file(f): if args.mode == _MODE_QSTR: name = match.replace("MP_QSTR_", "") output.append("Q(" + name + ")") - elif args.mode in (_MODE_COMPRESS, _MODE_MODULE): + elif args.mode in (_MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER): output.append(match) if last_fname: @@ -148,6 +153,8 @@ def cat_together(): mode_full = "Compressed data" elif args.mode == _MODE_MODULE: mode_full = "Module registrations" + elif args.mode == _MODE_ROOT_POINTER: + mode_full = "Root pointer registrations" if old_hash != new_hash: print(mode_full, "updated") try: @@ -208,7 +215,7 @@ if __name__ == "__main__": args.output_dir = sys.argv[4] args.output_file = None if len(sys.argv) == 5 else sys.argv[5] # Unused for command=split - if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE): + if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER): print("error: mode %s unrecognised" % sys.argv[2]) sys.exit(2) diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 4d0bbd22a..d7be0f934 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -11,6 +11,9 @@ set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h") set(MICROPY_MODULEDEFS_SPLIT "${MICROPY_GENHDR_DIR}/moduledefs.split") set(MICROPY_MODULEDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/moduledefs.collected") set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h") +set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split") +set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected") +set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h") # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen # manifest handling is at the end of this file. @@ -46,6 +49,7 @@ target_sources(${MICROPY_TARGET} PRIVATE ${MICROPY_MPVERSION} ${MICROPY_QSTRDEFS_GENERATED} ${MICROPY_MODULEDEFS} + ${MICROPY_ROOT_POINTERS} ) # Command to force the build of another command @@ -139,6 +143,31 @@ add_custom_command( DEPENDS ${MICROPY_MODULEDEFS_COLLECTED} ) +# Generate root_pointers.h + +add_custom_command( + OUTPUT ${MICROPY_ROOT_POINTERS_SPLIT} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split root_pointer ${MICROPY_GENHDR_DIR}/qstr.i.last ${MICROPY_GENHDR_DIR}/root_pointer _ + COMMAND touch ${MICROPY_ROOT_POINTERS_SPLIT} + DEPENDS ${MICROPY_QSTRDEFS_LAST} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_ROOT_POINTERS_COLLECTED} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat root_pointer _ ${MICROPY_GENHDR_DIR}/root_pointer ${MICROPY_ROOT_POINTERS_COLLECTED} + DEPENDS ${MICROPY_ROOT_POINTERS_SPLIT} + VERBATIM + COMMAND_EXPAND_LISTS +) + +add_custom_command( + OUTPUT ${MICROPY_ROOT_POINTERS} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/make_root_pointers.py ${MICROPY_ROOT_POINTERS_COLLECTED} > ${MICROPY_ROOT_POINTERS} + DEPENDS ${MICROPY_ROOT_POINTERS_COLLECTED} ${MICROPY_PY_DIR}/make_root_pointers.py +) + # Build frozen code if enabled if(MICROPY_FROZEN_MANIFEST) @@ -174,6 +203,7 @@ if(MICROPY_FROZEN_MANIFEST) COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} -v "MPY_DIR=${MICROPY_DIR}" -v "MPY_LIB_DIR=${MICROPY_LIB_DIR}" -v "PORT_DIR=${MICROPY_PORT_DIR}" -v "BOARD_DIR=${MICROPY_BOARD_DIR}" -b "${CMAKE_BINARY_DIR}" -f${MICROPY_CROSS_FLAGS} ${MICROPY_FROZEN_MANIFEST} DEPENDS MICROPY_FORCE_BUILD ${MICROPY_QSTRDEFS_GENERATED} + ${MICROPY_ROOT_POINTERS} ${MICROPY_MPYCROSS_DEPENDENCY} VERBATIM ) diff --git a/py/mkrules.mk b/py/mkrules.mk index fa1aad881..a7c437386 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -7,8 +7,8 @@ endif # Extra deps that need to happen before object compilation. OBJ_EXTRA_ORDER_DEPS = -# Generate moduledefs.h. -OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h +# Generate header files. +OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h $(HEADER_BUILD)/root_pointers.h ifeq ($(MICROPY_ROM_TEXT_COMPRESSION),1) # If compression is enabled, trigger the build of compressed.data.h... @@ -126,6 +126,16 @@ $(HEADER_BUILD)/moduledefs.collected: $(HEADER_BUILD)/moduledefs.split $(ECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat module _ $(HEADER_BUILD)/module $@ +# Module definitions via MP_REGISTER_ROOT_POINTER. +$(HEADER_BUILD)/root_pointers.split: $(HEADER_BUILD)/qstr.i.last + $(ECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split root_pointer $< $(HEADER_BUILD)/root_pointer _ + $(Q)$(TOUCH) $@ + +$(HEADER_BUILD)/root_pointers.collected: $(HEADER_BUILD)/root_pointers.split + $(ECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat root_pointer _ $(HEADER_BUILD)/root_pointer $@ + # Compressed error strings. $(HEADER_BUILD)/compressed.split: $(HEADER_BUILD)/qstr.i.last $(ECHO) "GEN $@" @@ -165,7 +175,7 @@ endif ifneq ($(FROZEN_MANIFEST),) # to build frozen_content.c from a manifest -$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) +$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h | $(MICROPY_MPYCROSS_DEPENDENCY) $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) endif diff --git a/py/mpstate.h b/py/mpstate.h index 98aa9a849..07a4f3860 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -192,6 +192,13 @@ typedef struct _mp_state_vm_t { // include any root pointers defined by a port MICROPY_PORT_ROOT_POINTERS + // Include any root pointers registered with MP_REGISTER_ROOT_POINTER(). + #ifndef NO_QSTR + // Only include root pointer definitions when not doing qstr extraction, because + // the qstr extraction stage also generates the root pointers header file. + #include "genhdr/root_pointers.h" + #endif + // root pointers for extmod #if MICROPY_REPL_EVENT_DRIVEN diff --git a/py/obj.h b/py/obj.h index c94d09afa..7ba4c0a5f 100644 --- a/py/obj.h +++ b/py/obj.h @@ -416,13 +416,18 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; #define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name} #define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name} +#ifndef NO_QSTR + // Declare a module as a builtin, processed by makemoduledefs.py // param module_name: MP_QSTR_ // param obj_module: mp_obj_module_t instance - -#ifndef NO_QSTR #define MP_REGISTER_MODULE(module_name, obj_module) -#endif + +// Declare a root pointer (to avoid garbage collection of a global static variable). +// param variable_declaration: a valid C variable declaration +#define MP_REGISTER_ROOT_POINTER(variable_declaration) + +#endif // NO_QSTR // Underlying map/hash table implementation (not dict object or map function) diff --git a/py/py.mk b/py/py.mk index dacfa1bd0..8aac460b4 100644 --- a/py/py.mk +++ b/py/py.mk @@ -218,6 +218,11 @@ $(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected @$(ECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@ +# build a list of registered root pointers for py/mpstate.h. +$(HEADER_BUILD)/root_pointers.h: $(HEADER_BUILD)/root_pointers.collected $(PY_SRC)/make_root_pointers.py + @$(ECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/make_root_pointers.py $< > $@ + # Standard C functions like memset need to be compiled with special flags so # the compiler does not optimise these functions in terms of themselves. CFLAGS_BUILTIN ?= -ffreestanding -fno-builtin -fno-lto