py: Allow registration of modules at their definition.

During make, makemoduledefs.py parses the current builds c files for
MP_REGISTER_MODULE(module_name, obj_module, enabled_define)

These are used to generate a header with the required entries for
"mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
This commit is contained in:
Andrew Leech 2019-02-18 14:58:44 +11:00 committed by Damien George
parent e4ac104b7f
commit cf22f4793c
6 changed files with 143 additions and 4 deletions

View File

@ -4,7 +4,7 @@
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
</Target>
<PropertyGroup>
@ -83,6 +83,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
</Target>
<Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
<PropertyGroup>
<DestFile>$(DestDir)moduledefs.h</DestFile>
<TmpFile>$(DestFile).tmp</TmpFile>
</PropertyGroup>
<ItemGroup>
<PyUserModuleFiles Include="@(ClCompile)">
<Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
</PyUserModuleFiles>
</ItemGroup>
<Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
<PropertyGroup>
<TmpFile>$(QstrGen).tmp</TmpFile>

107
py/makemoduledefs.py Normal file
View File

@ -0,0 +1,107 @@
#!/usr/bin/env python
# This pre-processor parses provided objects' c files for
# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
# These are used to generate a header with the required entries for
# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
from __future__ import print_function
import re
import os
import argparse
pattern = re.compile(
r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
flags=re.DOTALL
)
def find_c_file(obj_file, vpath):
""" Search vpaths for the c file that matches the provided object_file.
:param str obj_file: object file to find the matching c file for
:param List[str] vpath: List of base paths, similar to gcc vpath
:return: str path to c file or None
"""
c_file = None
relative_c_file = os.path.splitext(obj_file)[0] + ".c"
relative_c_file = relative_c_file.lstrip('/\\')
for p in vpath:
possible_c_file = os.path.join(p, relative_c_file)
if os.path.exists(possible_c_file):
c_file = possible_c_file
break
return c_file
def find_module_registrations(c_file):
""" Find any MP_REGISTER_MODULE definitions in the provided c file.
:param str c_file: path to c file to check
:return: List[(module_name, obj_module, enabled_define)]
"""
global pattern
if c_file is None:
# No c file to match the object file, skip
return set()
with open(c_file) as c_file_obj:
return set(re.findall(pattern, c_file_obj.read()))
def generate_module_table_header(modules):
""" Generate header with module table entries for builtin modules.
:param List[(module_name, obj_module, enabled_define)] modules: module defs
:return: None
"""
# Print header file for all external modules.
mod_defs = []
print("// Automatically generated by makemoduledefs.py.\n")
for module_name, obj_module, enabled_define in modules:
mod_def = "MODULE_DEF_{}".format(module_name.upper())
mod_defs.append(mod_def)
print((
"#if ({enabled_define})\n"
" extern const struct _mp_obj_module_t {obj_module};\n"
" #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
"#else\n"
" #define {mod_def}\n"
"#endif\n"
).format(module_name=module_name, obj_module=obj_module,
enabled_define=enabled_define, mod_def=mod_def)
)
print("\n#define MICROPY_REGISTERED_MODULES \\")
for mod_def in mod_defs:
print(" {mod_def} \\".format(mod_def=mod_def))
print("// MICROPY_REGISTERED_MODULES")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--vpath", default=".",
help="comma separated list of folders to search for c files in")
parser.add_argument("files", nargs="*",
help="list of c files to search")
args = parser.parse_args()
vpath = [p.strip() for p in args.vpath.split(',')]
modules = set()
for obj_file in args.files:
c_file = find_c_file(obj_file, vpath)
modules |= find_module_registrations(c_file)
generate_module_table_header(sorted(modules))
if __name__ == '__main__':
main()

View File

@ -40,4 +40,6 @@ const mp_obj_module_t mp_module_array = {
.globals = (mp_obj_dict_t*)&mp_module_array_globals,
};
MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY);
#endif

View File

@ -326,6 +326,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
#define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
#define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
// Declare a module as a builtin, processed by makemoduledefs.py
// param module_name: MP_QSTR_<module name>
// param obj_module: mp_obj_module_t instance
// prarm enabled_define: used as `#if (enabled_define) around entry`
#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
// Underlying map/hash table implementation (not dict object or map function)
typedef struct _mp_map_elem_t {

View File

@ -31,6 +31,8 @@
#include "py/runtime.h"
#include "py/builtin.h"
#include "genhdr/moduledefs.h"
STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
(void)kind;
mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in);
@ -136,9 +138,6 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
{ MP_ROM_QSTR(MP_QSTR_builtins), MP_ROM_PTR(&mp_module_builtins) },
{ MP_ROM_QSTR(MP_QSTR_micropython), MP_ROM_PTR(&mp_module_micropython) },
#if MICROPY_PY_ARRAY
{ MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_module_array) },
#endif
#if MICROPY_PY_IO
{ MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) },
#endif
@ -226,6 +225,11 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
// extra builtin modules as defined by a port
MICROPY_PORT_BUILTIN_MODULES
#ifdef MICROPY_REGISTERED_MODULES
// builtin modules declared with MP_REGISTER_MODULE()
MICROPY_REGISTERED_MODULES
#endif
};
MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);

View File

@ -323,6 +323,11 @@ $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_C
$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
# build a list of registered modules for py/objmodule.c.
$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
@$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
# Force nlr code to always be compiled with space-saving optimisation so
# that the function preludes are of a minimal and predictable form.
$(PY_BUILD)/nlr%.o: CFLAGS += -Os