Merge pull request adafruit#1754 from dmazzella/dmazzella-patch-1

add support for USER_C_MODULES

Author: tannewt

ports/nrf/Makefile

@@ -209,6 +209,7 @@ ifeq ($(INTERNAL_LIBM),1)
 OBJ += $(addprefix $(BUILD)/, $(SRC_LIBM:.c=.o))
 endif
 OBJ += $(addprefix $(BUILD)/, $(SRC_S:.s=.o))
+OBJ += $(addprefix $(BUILD)/, $(SRC_MOD:.c=.o))
 
 $(BUILD)/$(FATFS_DIR)/ff.o: COPT += -Os
 $(filter $(PY_BUILD)/../extmod/vfs_fat_%.o, $(PY_O)): COPT += -Os

py/makemoduledefs.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+# This pre-processor parses provided objects' c files for
+# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+# These are used to generate a header with the required entries for
+# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
+
+from __future__ import print_function
+
+import re
+import os
+import argparse
+
+
+pattern = re.compile(
+    r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
+    flags=re.DOTALL
+)
+
+
+def find_c_file(obj_file, vpath):
+    """ Search vpaths for the c file that matches the provided object_file.
+
+    :param str obj_file: object file to find the matching c file for
+    :param List[str] vpath: List of base paths, similar to gcc vpath
+    :return: str path to c file or None
+    """
+    c_file = None
+    relative_c_file = os.path.splitext(obj_file)[0] + ".c"
+    relative_c_file = relative_c_file.lstrip('/\\')
+    for p in vpath:
+        possible_c_file = os.path.join(p, relative_c_file)
+        if os.path.exists(possible_c_file):
+            c_file = possible_c_file
+            break
+
+    return c_file
+
+
+def find_module_registrations(c_file):
+    """ Find any MP_REGISTER_MODULE definitions in the provided c file.
+
+    :param str c_file: path to c file to check
+    :return: List[(module_name, obj_module, enabled_define)]
+    """
+    global pattern
+
+    if c_file is None:
+        # No c file to match the object file, skip
+        return set()
+
+    with open(c_file) as c_file_obj:
+        return set(re.findall(pattern, c_file_obj.read()))
+
+
+def generate_module_table_header(modules):
+    """ Generate header with module table entries for builtin modules.
+
+    :param List[(module_name, obj_module, enabled_define)] modules: module defs
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    mod_defs = []
+    print("// Automatically generated by makemoduledefs.py.\n")
+    for module_name, obj_module, enabled_define in modules:
+        mod_def = "MODULE_DEF_{}".format(module_name.upper())
+        mod_defs.append(mod_def)
+        print((
+            "#if ({enabled_define})\n"
+            "    extern const struct _mp_obj_module_t {obj_module};\n"
+            "    #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
+            "#else\n"
+            "    #define {mod_def}\n"
+            "#endif\n"
+            ).format(module_name=module_name, obj_module=obj_module,
+                     enabled_define=enabled_define, mod_def=mod_def)
+        )
+
+    print("\n#define MICROPY_REGISTERED_MODULES \\")
+
+    for mod_def in mod_defs:
+        print("    {mod_def} \\".format(mod_def=mod_def))
+
+    print("// MICROPY_REGISTERED_MODULES")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vpath", default=".",
+                        help="comma separated list of folders to search for c files in")
+    parser.add_argument("files", nargs="*",
+                        help="list of c files to search")
+    args = parser.parse_args()
+
+    vpath = [p.strip() for p in args.vpath.split(',')]
+
+    modules = set()
+    for obj_file in args.files:
+        c_file = find_c_file(obj_file, vpath)
+        modules |= find_module_registrations(c_file)
+
+    generate_module_table_header(sorted(modules))
+
+
+if __name__ == '__main__':
+    main()

py/mkrules.mk

@@ -20,12 +20,12 @@ endif
 # can be located. By following this scheme, it allows a single build rule
 # to be used to compile all .c files.
 
-vpath %.S . $(TOP)
+vpath %.S . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.S
 	$(STEPECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
-vpath %.s . $(TOP)
+vpath %.s . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.s
 	$(STEPECHO) "AS $<"
 	$(Q)$(AS) -o $@ $<
@@ -42,7 +42,7 @@ $(Q)$(CC) $(CFLAGS) -c -MD -o $@ $<
   $(RM) -f $(@:.o=.d)
 endef
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.c
 	$(call compile_c)
 
@@ -56,7 +56,7 @@ $(BUILD)/%.o: %.c
 
 QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 
 $(BUILD)/%.pp: %.c
 	$(STEPECHO) "PreProcess $<"

py/obj.h

@@ -338,6 +338,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
 #define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
 #define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
 
+// Declare a module as a builtin, processed by makemoduledefs.py
+// param module_name: MP_QSTR_<module name>
+// param obj_module: mp_obj_module_t instance
+// prarm enabled_define: used as `#if (enabled_define) around entry`
+
+#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+
 // Underlying map/hash table implementation (not dict object or map function)
 
 typedef struct _mp_map_elem_t {

py/objmodule.c

@@ -32,6 +32,8 @@
 #include "py/runtime.h"
 #include "py/builtin.h"
 
+#include "genhdr/moduledefs.h"
+
 STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
     mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in);
@@ -252,6 +254,11 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
     // extra builtin modules as defined by a port
     MICROPY_PORT_BUILTIN_MODULES
 
+    #ifdef MICROPY_REGISTERED_MODULES
+    // builtin modules declared with MP_REGISTER_MODULE()
+    MICROPY_REGISTERED_MODULES
+    #endif
+
 #if defined(MICROPY_DEBUG_MODULES) && defined(MICROPY_PORT_BUILTIN_DEBUG_MODULES)
     , MICROPY_PORT_BUILTIN_DEBUG_MODULES
 #endif

py/py.mk

@@ -105,6 +105,24 @@ $(BUILD)/$(BTREE_DIR)/%.o: CFLAGS += -Wno-old-style-definition -Wno-sign-compare
 $(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS)
 endif
 
+# External modules written in C.
+ifneq ($(USER_C_MODULES),)
+# pre-define USERMOD variables as expanded so that variables are immediate 
+# expanded as they're added to them
+SRC_USERMOD := 
+CFLAGS_USERMOD :=
+LDFLAGS_USERMOD :=
+$(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \
+    $(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\
+    $(info Including User C Module from $(USERMOD_DIR))\
+	$(eval include $(module))\
+)
+
+SRC_MOD += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD))
+CFLAGS_MOD += $(CFLAGS_USERMOD)
+LDFLAGS_MOD += $(LDFLAGS_USERMOD)
+endif
+
 # py object files
 PY_CORE_O_BASENAME = $(addprefix py/,\
 	mpstate.o \
@@ -287,6 +305,13 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 	$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeversionhdr.py $@
 
+# build a list of registered modules for py/objmodule.c.
+$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
+	@$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
+
+SRC_QSTR += $(HEADER_BUILD)/moduledefs.h
+
 # mpconfigport.mk is optional, but changes to it may drastically change
 # overall config, so they need to be caught
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)