py/make_root_pointers: Add MP_REGISTER_ROOT_POINTER parser/generator.
This adds new compile-time infrastructure to parse source code files for `MP_REGISTER_ROOT_POINTER()` and generates a new `root_pointers.h` header file containing the collected declarations. This works the same as the existing `MP_REGISTER_MODULE()` feature. Signed-off-by: David Lechner <david@pybricks.com>
This commit is contained in:
David Lechner
Damien George
parent
a8d78cc398
commit
fc3d7ae11b
|
|
@ -134,3 +134,7 @@ $(HEADER_BUILD)/mpversion.h: | $(HEADER_BUILD)
|
|||
# Create an empty "moduledefs.h" needed by py/mkrules.mk
|
||||
$(HEADER_BUILD)/moduledefs.h: | $(HEADER_BUILD)
|
||||
touch $@
|
||||
|
||||
# Create an empty "root_pointers.h" needed by py/mkrules.mk
|
||||
$(HEADER_BUILD)/root_pointers.h: | $(HEADER_BUILD)
|
||||
touch $@
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ PREFIX_FILE = ../boards/stm32f4xx_prefix.c
|
|||
BOARD_PINS = $(BOARD_DIR)/pins.csv
|
||||
HEADER_BUILD = $(BUILD)/genhdr
|
||||
GEN_QSTRDEFS_GENERATED = $(HEADER_BUILD)/qstrdefs.generated.h
|
||||
GEN_ROOT_POINTERS = $(HEADER_BUILD)/root_pointers.h
|
||||
GEN_PINS_SRC = $(BUILD)/pins_$(BOARD).c
|
||||
GEN_PINS_HDR = $(HEADER_BUILD)/pins.h
|
||||
GEN_PINS_QSTR = $(HEADER_BUILD)/pins_qstr.h
|
||||
|
|
@ -220,7 +221,7 @@ GEN_PINS_AF_CONST = $(HEADER_BUILD)/pins_af_const.h
|
|||
GEN_PINS_AF_DEFS = $(HEADER_BUILD)/pins_af_defs.h
|
||||
GEN_PINS_AF_PY = $(BUILD)/pins_af.py
|
||||
|
||||
$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_PINS_AF_DEFS)
|
||||
$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_ROOT_POINTERS) $(GEN_PINS_AF_DEFS)
|
||||
|
||||
$(HEADER_BUILD):
|
||||
$(MKDIR) -p $(BUILD)/genhdr
|
||||
|
|
@ -228,6 +229,9 @@ $(HEADER_BUILD):
|
|||
$(GEN_QSTRDEFS_GENERATED): | $(HEADER_BUILD)
|
||||
$(Q)echo "// empty" > $@
|
||||
|
||||
$(GEN_ROOT_POINTERS): | $(HEADER_BUILD)
|
||||
$(Q)echo "// empty" > $@
|
||||
|
||||
$(GEN_PINS_AF_DEFS): $(BOARD_PINS) $(MAKE_PINS) ../$(AF_FILE) $(PREFIX_FILE) | $(HEADER_BUILD)
|
||||
$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(MAKE_PINS) --board $(BOARD_PINS) --af ../$(AF_FILE) \
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
|
||||
|
||||
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
|
||||
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
|
||||
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeRootPointers;MakeQstrData">
|
||||
</Target>
|
||||
|
||||
<PropertyGroup>
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
<QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
|
||||
<QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
|
||||
<ModuleDefsCollected>$(DestDir)/moduledefs.collected</ModuleDefsCollected>
|
||||
<RootPointersCollected>$(DestDir)/root_pointers.collected</RootPointersCollected>
|
||||
<PyPython Condition="'$(PyPython)' == ''">$(MICROPY_CPYTHON3)</PyPython>
|
||||
<PyPython Condition="'$(PyPython)' == ''">python</PyPython>
|
||||
<CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
|
||||
|
|
@ -45,7 +46,7 @@
|
|||
using(var outFile = System.IO.File.CreateText(OutputFile)) {
|
||||
foreach(var inFile in InputFiles)
|
||||
foreach(var line in System.IO.File.ReadAllLines(inFile))
|
||||
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
|
||||
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR") || line.Contains("MP_REGISTER"))
|
||||
outFile.WriteLine( line );
|
||||
}
|
||||
]]>
|
||||
|
|
@ -114,6 +115,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
|
|||
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
|
||||
</Target>
|
||||
|
||||
<Target Name="CollectRootPointers" DependsOnTargets="MakeQstrDefs" Inputs="$(DestDir)qstr.i.last" Outputs="$(RootPointersCollected)">
|
||||
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split root_pointer $(DestDir)qstr.i.last $(DestDir)root_pointer _"/>
|
||||
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat root_pointer _ $(DestDir)root_pointer $(RootPointersCollected)"/>
|
||||
</Target>
|
||||
|
||||
<Target Name="MakeRootPointers" DependsOnTargets="CollectRootPointers" Inputs="$(PySrcDir)make_root_pointers.py;$(RootPointersCollected)" Outputs="$(DestDir)root_pointers.h">
|
||||
<PropertyGroup>
|
||||
<DestFile>$(DestDir)root_pointers.h</DestFile>
|
||||
<TmpFile>$(DestFile).tmp</TmpFile>
|
||||
</PropertyGroup>
|
||||
<Exec Command="$(PyPython) $(PySrcDir)make_root_pointers.py $(RootPointersCollected) > $(TmpFile)"/>
|
||||
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
|
||||
</Target>
|
||||
|
||||
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
|
||||
<PropertyGroup>
|
||||
<TmpFile>$(QstrGen).tmp</TmpFile>
|
||||
|
|
@ -132,7 +147,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
|
|||
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
|
||||
</Target>
|
||||
|
||||
<Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
|
||||
<Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData;MakeRootPointers" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
|
||||
<ItemGroup>
|
||||
<ClCompile Include="$(PyBuildDir)frozen_content.c"/>
|
||||
<ClCompile>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
"""
|
||||
This pre-processor parses a single file containing a list of
|
||||
MP_REGISTER_ROOT_POINTER(variable declaration) items.
|
||||
|
||||
These are used to generate a header with the required entries for
|
||||
"struct _mp_state_vm_t" in py/mpstate.h
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import re
|
||||
|
||||
|
||||
PATTERN = re.compile(r"MP_REGISTER_ROOT_POINTER\((.*?)\);")
|
||||
|
||||
|
||||
def find_root_pointer_registrations(filename):
|
||||
"""Find any MP_REGISTER_ROOT_POINTER definitions in the provided file.
|
||||
|
||||
:param str filename: path to file to check
|
||||
:return: List[variable_declaration]
|
||||
"""
|
||||
with io.open(filename, encoding="utf-8") as c_file_obj:
|
||||
return set(re.findall(PATTERN, c_file_obj.read()))
|
||||
|
||||
|
||||
def generate_root_pointer_header(root_pointers):
|
||||
"""Generate header with root pointer entries.
|
||||
|
||||
:param List[variable_declaration] root_pointers: root pointer declarations
|
||||
:return: None
|
||||
"""
|
||||
|
||||
# Print header file for all external modules.
|
||||
print("// Automatically generated by make_root_pointers.py.")
|
||||
print()
|
||||
|
||||
for item in root_pointers:
|
||||
print(item, end=";")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("file", nargs=1, help="file with MP_REGISTER_ROOT_POINTER definitions")
|
||||
args = parser.parse_args()
|
||||
|
||||
root_pointers = find_root_pointer_registrations(args.file[0])
|
||||
generate_root_pointer_header(sorted(root_pointers))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -24,6 +24,9 @@ _MODE_COMPRESS = "compress"
|
|||
# Extract MP_REGISTER_MODULE(...) macros.
|
||||
_MODE_MODULE = "module"
|
||||
|
||||
# Extract MP_REGISTER_ROOT_POINTER(...) macros.
|
||||
_MODE_ROOT_POINTER = "root_pointer"
|
||||
|
||||
|
||||
def is_c_source(fname):
|
||||
return os.path.splitext(fname)[1] in [".c"]
|
||||
|
|
@ -90,6 +93,8 @@ def process_file(f):
|
|||
re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)')
|
||||
elif args.mode == _MODE_MODULE:
|
||||
re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);")
|
||||
elif args.mode == _MODE_ROOT_POINTER:
|
||||
re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);")
|
||||
output = []
|
||||
last_fname = None
|
||||
for line in f:
|
||||
|
|
@ -111,7 +116,7 @@ def process_file(f):
|
|||
if args.mode == _MODE_QSTR:
|
||||
name = match.replace("MP_QSTR_", "")
|
||||
output.append("Q(" + name + ")")
|
||||
elif args.mode in (_MODE_COMPRESS, _MODE_MODULE):
|
||||
elif args.mode in (_MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
|
||||
output.append(match)
|
||||
|
||||
if last_fname:
|
||||
|
|
@ -148,6 +153,8 @@ def cat_together():
|
|||
mode_full = "Compressed data"
|
||||
elif args.mode == _MODE_MODULE:
|
||||
mode_full = "Module registrations"
|
||||
elif args.mode == _MODE_ROOT_POINTER:
|
||||
mode_full = "Root pointer registrations"
|
||||
if old_hash != new_hash:
|
||||
print(mode_full, "updated")
|
||||
try:
|
||||
|
|
@ -208,7 +215,7 @@ if __name__ == "__main__":
|
|||
args.output_dir = sys.argv[4]
|
||||
args.output_file = None if len(sys.argv) == 5 else sys.argv[5] # Unused for command=split
|
||||
|
||||
if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE):
|
||||
if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
|
||||
print("error: mode %s unrecognised" % sys.argv[2])
|
||||
sys.exit(2)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h")
|
|||
set(MICROPY_MODULEDEFS_SPLIT "${MICROPY_GENHDR_DIR}/moduledefs.split")
|
||||
set(MICROPY_MODULEDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/moduledefs.collected")
|
||||
set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h")
|
||||
set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split")
|
||||
set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected")
|
||||
set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h")
|
||||
|
||||
# Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen
|
||||
# manifest handling is at the end of this file.
|
||||
|
|
@ -46,6 +49,7 @@ target_sources(${MICROPY_TARGET} PRIVATE
|
|||
${MICROPY_MPVERSION}
|
||||
${MICROPY_QSTRDEFS_GENERATED}
|
||||
${MICROPY_MODULEDEFS}
|
||||
${MICROPY_ROOT_POINTERS}
|
||||
)
|
||||
|
||||
# Command to force the build of another command
|
||||
|
|
@ -139,6 +143,31 @@ add_custom_command(
|
|||
DEPENDS ${MICROPY_MODULEDEFS_COLLECTED}
|
||||
)
|
||||
|
||||
# Generate root_pointers.h
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${MICROPY_ROOT_POINTERS_SPLIT}
|
||||
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split root_pointer ${MICROPY_GENHDR_DIR}/qstr.i.last ${MICROPY_GENHDR_DIR}/root_pointer _
|
||||
COMMAND touch ${MICROPY_ROOT_POINTERS_SPLIT}
|
||||
DEPENDS ${MICROPY_QSTRDEFS_LAST}
|
||||
VERBATIM
|
||||
COMMAND_EXPAND_LISTS
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${MICROPY_ROOT_POINTERS_COLLECTED}
|
||||
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat root_pointer _ ${MICROPY_GENHDR_DIR}/root_pointer ${MICROPY_ROOT_POINTERS_COLLECTED}
|
||||
DEPENDS ${MICROPY_ROOT_POINTERS_SPLIT}
|
||||
VERBATIM
|
||||
COMMAND_EXPAND_LISTS
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${MICROPY_ROOT_POINTERS}
|
||||
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/make_root_pointers.py ${MICROPY_ROOT_POINTERS_COLLECTED} > ${MICROPY_ROOT_POINTERS}
|
||||
DEPENDS ${MICROPY_ROOT_POINTERS_COLLECTED} ${MICROPY_PY_DIR}/make_root_pointers.py
|
||||
)
|
||||
|
||||
# Build frozen code if enabled
|
||||
|
||||
if(MICROPY_FROZEN_MANIFEST)
|
||||
|
|
@ -174,6 +203,7 @@ if(MICROPY_FROZEN_MANIFEST)
|
|||
COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} -v "MPY_DIR=${MICROPY_DIR}" -v "MPY_LIB_DIR=${MICROPY_LIB_DIR}" -v "PORT_DIR=${MICROPY_PORT_DIR}" -v "BOARD_DIR=${MICROPY_BOARD_DIR}" -b "${CMAKE_BINARY_DIR}" -f${MICROPY_CROSS_FLAGS} ${MICROPY_FROZEN_MANIFEST}
|
||||
DEPENDS MICROPY_FORCE_BUILD
|
||||
${MICROPY_QSTRDEFS_GENERATED}
|
||||
${MICROPY_ROOT_POINTERS}
|
||||
${MICROPY_MPYCROSS_DEPENDENCY}
|
||||
VERBATIM
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ endif
|
|||
# Extra deps that need to happen before object compilation.
|
||||
OBJ_EXTRA_ORDER_DEPS =
|
||||
|
||||
# Generate moduledefs.h.
|
||||
OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h
|
||||
# Generate header files.
|
||||
OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h $(HEADER_BUILD)/root_pointers.h
|
||||
|
||||
ifeq ($(MICROPY_ROM_TEXT_COMPRESSION),1)
|
||||
# If compression is enabled, trigger the build of compressed.data.h...
|
||||
|
|
@ -126,6 +126,16 @@ $(HEADER_BUILD)/moduledefs.collected: $(HEADER_BUILD)/moduledefs.split
|
|||
$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat module _ $(HEADER_BUILD)/module $@
|
||||
|
||||
# Module definitions via MP_REGISTER_ROOT_POINTER.
|
||||
$(HEADER_BUILD)/root_pointers.split: $(HEADER_BUILD)/qstr.i.last
|
||||
$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split root_pointer $< $(HEADER_BUILD)/root_pointer _
|
||||
$(Q)$(TOUCH) $@
|
||||
|
||||
$(HEADER_BUILD)/root_pointers.collected: $(HEADER_BUILD)/root_pointers.split
|
||||
$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat root_pointer _ $(HEADER_BUILD)/root_pointer $@
|
||||
|
||||
# Compressed error strings.
|
||||
$(HEADER_BUILD)/compressed.split: $(HEADER_BUILD)/qstr.i.last
|
||||
$(ECHO) "GEN $@"
|
||||
|
|
@ -165,7 +175,7 @@ endif
|
|||
|
||||
ifneq ($(FROZEN_MANIFEST),)
|
||||
# to build frozen_content.c from a manifest
|
||||
$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY)
|
||||
$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h | $(MICROPY_MPYCROSS_DEPENDENCY)
|
||||
$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST)
|
||||
endif
|
||||
|
||||
|
|
|
|||
|
|
@ -192,6 +192,13 @@ typedef struct _mp_state_vm_t {
|
|||
// include any root pointers defined by a port
|
||||
MICROPY_PORT_ROOT_POINTERS
|
||||
|
||||
// Include any root pointers registered with MP_REGISTER_ROOT_POINTER().
|
||||
#ifndef NO_QSTR
|
||||
// Only include root pointer definitions when not doing qstr extraction, because
|
||||
// the qstr extraction stage also generates the root pointers header file.
|
||||
#include "genhdr/root_pointers.h"
|
||||
#endif
|
||||
|
||||
// root pointers for extmod
|
||||
|
||||
#if MICROPY_REPL_EVENT_DRIVEN
|
||||
|
|
|
|||
11
py/obj.h
11
py/obj.h
|
|
@ -416,13 +416,18 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
|
|||
#define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
|
||||
#define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
|
||||
|
||||
#ifndef NO_QSTR
|
||||
|
||||
// Declare a module as a builtin, processed by makemoduledefs.py
|
||||
// param module_name: MP_QSTR_<module name>
|
||||
// param obj_module: mp_obj_module_t instance
|
||||
|
||||
#ifndef NO_QSTR
|
||||
#define MP_REGISTER_MODULE(module_name, obj_module)
|
||||
#endif
|
||||
|
||||
// Declare a root pointer (to avoid garbage collection of a global static variable).
|
||||
// param variable_declaration: a valid C variable declaration
|
||||
#define MP_REGISTER_ROOT_POINTER(variable_declaration)
|
||||
|
||||
#endif // NO_QSTR
|
||||
|
||||
// Underlying map/hash table implementation (not dict object or map function)
|
||||
|
||||
|
|
|
|||
5
py/py.mk
5
py/py.mk
|
|
@ -218,6 +218,11 @@ $(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected
|
|||
@$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@
|
||||
|
||||
# build a list of registered root pointers for py/mpstate.h.
|
||||
$(HEADER_BUILD)/root_pointers.h: $(HEADER_BUILD)/root_pointers.collected $(PY_SRC)/make_root_pointers.py
|
||||
@$(ECHO) "GEN $@"
|
||||
$(Q)$(PYTHON) $(PY_SRC)/make_root_pointers.py $< > $@
|
||||
|
||||
# Standard C functions like memset need to be compiled with special flags so
|
||||
# the compiler does not optimise these functions in terms of themselves.
|
||||
CFLAGS_BUILTIN ?= -ffreestanding -fno-builtin -fno-lto
|
||||
|
|
|
|||
Loading…
Reference in New Issue