diff --git a/.appveyor.yml b/.appveyor.yml index 70fbba2..bb1f4ff 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,17 +5,30 @@ environment: - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 MICROPY_CPYTHON3: c:/python39/python.exe FullTypeCheck: 0 + StaticPyObjRootPtr: 0 + BuildMsys2Version: 1 + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 + MICROPY_CPYTHON3: c:/python39/python.exe + FullTypeCheck: 0 + StaticPyObjRootPtr: 1 BuildMsys2Version: 1 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 MICROPY_CPYTHON3: c:/python38/python.exe FullTypeCheck: 1 + StaticPyObjRootPtr: 0 BuildMsys2Version: 0 - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu MICROPY_CPYTHON3: python3 FullTypeCheck: 0 + StaticPyObjRootPtr: 0 + - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu + MICROPY_CPYTHON3: python3 + FullTypeCheck: 1 + StaticPyObjRootPtr: 0 - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu MICROPY_CPYTHON3: python3 FullTypeCheck: 1 + StaticPyObjRootPtr: 1 configuration: - Debug @@ -99,7 +112,7 @@ for: C:\msys64\usr\bin\bash.exe -l -c "make V=1 MICROPYTHON_PORT_DIR=../micropython/ports/windows clean" # The redirection is because 'make submodules' might get called, which in turn calls git commands which write to stderr # and that is considered an error on Appveyor. And GIT_REDIRECT_STDERR seems to have no effect on the git version used in this msys2. - &{C:\msys64\usr\bin\bash.exe -l -c "make V=1 PYTHON=/usr/bin/python3 MICROPYTHON_PORT_DIR=../micropython/ports/windows test"} 2>&1 + &{C:\msys64\usr\bin\bash.exe -l -c "make V=1 CPPFLAGS_EXTRA='-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck -DUPYWRAP_STATICPYOBJ_USE_ROOTPTR=$env:StaticPyObjRootPtr' PYTHON=/usr/bin/python3 MICROPYTHON_PORT_DIR=../micropython/ports/windows test"} 2>&1 if ($LASTEXITCODE -ne 0) { throw "MSYS2 build exited with code $LASTEXITCODE" } @@ -112,7 +125,7 @@ for: build_script: - ps: | - make test CC=gcc-9 CXX=g++-9 CPPFLAGS_EXTRA="-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck" + make test CC=gcc-9 CXX=g++-9 CPPFLAGS_EXTRA="-DUPYWRAP_FULLTYPECHECK=$env:FullTypeCheck -DUPYWRAP_STATICPYOBJ_USE_ROOTPTR=$env:StaticPyObjRootPtr" on_failure: - ps: | diff --git a/Makefile b/Makefile index 819bf67..6212719 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ endif V ?= 0 MAKEOPTS ?= -j4 V=$(V) VARIANT=$(VARIANT) VARIANT_DIR=$(VARIANT_DIR) MAKEUPY = make -C $(MICROPYTHON_PORT_DIR) $(MAKEOPTS) -UPYFLAGS = MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 MICROPY_PY_AXTLS=0 MICROPY_FATFS=0 MICROPY_PY_THREAD=0 +UPYFLAGS = QSTR_GLOBAL_DEPENDENCIES=$(CURDIR)/mpy_wrap_root_pointer.c MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 MICROPY_PY_AXTLS=0 MICROPY_FATFS=0 MICROPY_PY_THREAD=0 # Extra features we need; note that for the ports tested so far this is already enabled or else # defined conditionally so we can just define it here again. Otherwise make this conditional diff --git a/README.md b/README.md index a3fe7d1..4c48e6c 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,21 @@ import foo print(foo.TransformList(['a', 'b'])) # Prints ['aTRANSFORM', 'bTRANSFORM'] ``` +When using the `UPYWRAP_STATICPYOBJ_USE_ROOTPTR` configuration option (opt-in): + +The micropython-wrap type table needs to be registered as root pointer. Safe this as a `.c` file and add it to micropythons +`MICROPY_SOURCE_QSTR` sources in cmake or `SRC_QSTR` in make. This is to make sure it is added to the global states root pointers +and thus not collected by micropythons garbage-collector. + +```c +#include "py/obj.h" + +//micropythons global storage needs to be registered as root pointer to avoid garbage collection +//the user needs to do this at least once to be able to compile micropython-wrap +MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage); +``` + + Type Conversion --------------- Conversion between standard native types and `mp_obj_t`, the MicroPython opaque object type diff --git a/detail/configuration.h b/detail/configuration.h index fc88f85..a572968 100644 --- a/detail/configuration.h +++ b/detail/configuration.h @@ -127,4 +127,15 @@ #define UPYWRAP_MAXNUMKWARGS (8) #endif +//Store the StaticPyObjectStore in a root pointer map instead of a random module's globals dict. +//Enabling this requires the user to add the micropython root pointer in one micropython source file: +// +// MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage); +// +//The benefit of this is that the StaticPyObjectStore is now hidden from the mpy runtime. +//This is disabled by default for backwards compatibility. +#ifndef UPYWRAP_STATICPYOBJ_USE_ROOTPTR +#define UPYWRAP_STATICPYOBJ_USE_ROOTPTR (0) +#endif + #endif diff --git a/detail/micropython.h b/detail/micropython.h index 73f87f4..340b44c 100644 --- a/detail/micropython.h +++ b/detail/micropython.h @@ -1,6 +1,7 @@ #ifndef MICROPYTHON_WRAP_DETAIL_MICROPYTHON_H #define MICROPYTHON_WRAP_DETAIL_MICROPYTHON_H +#include "mpmap.h" #include "micropythonc.h" #include #include @@ -11,6 +12,32 @@ namespace upywrap { +#if UPYWRAP_STATICPYOBJ_USE_ROOTPTR + /** + * @brief This is the global map in the micropython state which is used to store the class types and function pointers. + * + * All objects that micropython-wrap allocates are referenced by this map, or by maps in this map. + * It needs to be initialized only once, but can be reset by assigning a new empty map to it. + * This reset function can be used to reset the micropython-wrap state stored on the micropython heap. + * It MUST be called, if the micropython heap is reset externally. + */ + inline void reset_mpy_wrap_global_map() { + MP_STATE_VM(micropython_wrap_global_storage) = m_new0(mp_map_t, 1); + } + + inline mp_map_t* get_mpy_wrap_global_map() { + return MP_STATE_VM(micropython_wrap_global_storage); + } + + inline void init_mpy_wrap_global_map() { + static bool init = false; + if (!init) { + init = true; + reset_mpy_wrap_global_map(); + } + } +#endif + inline mp_obj_t new_qstr( qstr what ) { return MP_OBJ_NEW_QSTR( what ); @@ -367,6 +394,12 @@ namespace upywrap return *List(); } + static void ResetBackEnd() + { + // Other functions like InitBackEnd will check if it's already initialized + *List() = nullptr; + } + static bool Initialized() { return !!*List(); @@ -481,9 +514,23 @@ namespace upywrap */ inline void InitializePyObjectStore( mp_obj_dict_t& dict ) { - if( !StaticPyObjectStore::Initialized() ) +#if UPYWRAP_STATICPYOBJ_USE_ROOTPTR + (void) dict; + upywrap::init_mpy_wrap_global_map(); + MPyMapView mpy_wrap_map { get_mpy_wrap_global_map() }; +#else + if( StaticPyObjectStore::Initialized() ) { + return; + } + MPyMapView mpy_wrap_map { &dict.map }; +#endif + + qstr static_py_obj_store_key = qstr_from_str( "_StaticPyObjectStore" ); + if( !mpy_wrap_map.contains( static_py_obj_store_key ) ) { - mp_obj_dict_store( &dict, new_qstr( "_StaticPyObjectStore" ), StaticPyObjectStore::InitBackEnd() ); + // If we can't find it in the map, the mpy state is either fresh or was reset. Reinit from scratch. + StaticPyObjectStore::ResetBackEnd(); + mpy_wrap_map[static_py_obj_store_key] = StaticPyObjectStore::InitBackEnd(); } } diff --git a/detail/mpmap.h b/detail/mpmap.h new file mode 100644 index 0000000..d2dbd78 --- /dev/null +++ b/detail/mpmap.h @@ -0,0 +1,35 @@ +#pragma once + +#include "micropythonc.h" +#include + +namespace upywrap { +/** + * @brief A wrapper around the MicroPython map type. + * + * This class provides a convenient way to interact with the MicroPython map type. + * Objects of this type do not own the map, and are not responsible for its lifetime. + */ +template +class MPyMapView { + static_assert(std::is_pointer::value, "upywrap::MpyMapView: Value must be a pointer type."); + static_assert(std::is_same::value, "upywrap::MPyMapView: Currently only qstr keys are supported."); + +public: + using mapped_type = Value; + explicit MPyMapView(mp_map_t* map_ptr) : map_ { map_ptr } {} + + Value& operator[](const qstr key) { + mp_map_elem_t* elem = mp_map_lookup(map_, MP_OBJ_NEW_QSTR(key), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + return reinterpret_cast(elem->value); + } + + bool contains(const qstr key) const { + mp_map_elem_t* elem = mp_map_lookup(map_, MP_OBJ_NEW_QSTR(key), MP_MAP_LOOKUP); + return elem != nullptr; + } + +private: + mp_map_t* map_; +}; +} // namespace upywrap \ No newline at end of file diff --git a/mpy_wrap_root_pointer.c b/mpy_wrap_root_pointer.c new file mode 100644 index 0000000..b8dd87d --- /dev/null +++ b/mpy_wrap_root_pointer.c @@ -0,0 +1,7 @@ +//Add micropython-wrap as a "USERMOD" in micropython to register this root pointer +//Alternative, this line can also be copy-pasted to user code. +//Only required when using the `UPYWRAP_STATICPYOBJ_USE_ROOTPTR` configuration option (opt-in): +//Needs to be done once: Add global type map for micropython-wrap to the micropython state as root pointer +//This enables micropython-wrap to store anonymous type information while preventing visibility of the _StaticPyObjectStore +//in user modules. +MP_REGISTER_ROOT_POINTER(mp_map_t* micropython_wrap_global_storage); \ No newline at end of file