From bd585ae8244230a59099f3ace5a74fa35fc3a4fd Mon Sep 17 00:00:00 2001 From: David Heejong Park Date: Fri, 11 Oct 2024 23:24:12 +0200 Subject: [PATCH 1/4] Added Cython option --- pythonfmu/builder.py | 25 ++++- .../src/pythonfmu/PySlaveInstance.cpp | 104 ++++++++++++------ 2 files changed, 91 insertions(+), 38 deletions(-) diff --git a/pythonfmu/builder.py b/pythonfmu/builder.py index da9275d..e0f7ed0 100644 --- a/pythonfmu/builder.py +++ b/pythonfmu/builder.py @@ -8,12 +8,15 @@ import tempfile import zipfile import inspect +import glob from pathlib import Path from typing import Iterable, Optional, Tuple, Union from xml.dom.minidom import parseString from xml.etree.ElementTree import Element, SubElement, tostring from .osutil import get_lib_extension, get_platform from .fmi2slave import FMI2_MODEL_OPTIONS, Fmi2Slave +from setuptools import setup +from Cython.Build import cythonize FilePath = Union[str, Path] HERE = Path(__file__).parent @@ -78,6 +81,7 @@ def build_FMU( documentation_folder: Optional[FilePath] = None, **options, ) -> Path: + has_cythonize: bool = options["cythonize"] script_file = Path(script_file) if not script_file.exists(): raise ValueError(f"No such file {script_file!s}") @@ -100,7 +104,15 @@ def build_FMU( with tempfile.TemporaryDirectory(prefix="pythonfmu_") as tempd: temp_dir = Path(tempd) - shutil.copy2(script_file, temp_dir) + if has_cythonize: + setup( + script_args=["build_ext", "--inplace"], + ext_modules=cythonize(str(script_file)), + ) + for bin_file in glob.glob(f"{script_file.stem}*.{'pyd' if sys.platform == 'win32' else 'so'}"): + shutil.copy2(bin_file, temp_dir) + else: + shutil.copy2(script_file, temp_dir) # Embed pythonfmu in the FMU so it does not need to be included dep_folder = temp_dir / "pythonfmu" @@ -129,7 +141,7 @@ def build_FMU( shutil.copy2(file_, temp_dir) model_identifier, xml = get_model_description( - temp_dir.absolute() / script_file.name, module_name + script_file if cythonize else temp_dir.absolute() / script_file.name, module_name ) dest_file = dest / f"{model_identifier}.fmu" @@ -153,6 +165,11 @@ def build_FMU( # Add information for the Python loader zip_fmu.writestr(str(resource.joinpath("slavemodule.txt")), module_name) + if cythonize: + zip_fmu.writestr(str(resource.joinpath("filetype.txt")), "bin") + else: + zip_fmu.writestr(str(resource.joinpath("filetype.txt")), "script") + # Add FMI API wrapping Python class source source_node = SubElement(type_node, "SourceFiles") sources = Path("sources") @@ -245,3 +262,7 @@ def create_command_parser(parser: argparse.ArgumentParser): ) parser.set_defaults(execute=FmuBuilder.build_FMU) + + parser.add_argument( + "-c", "--cythonize", dest="cythonize", action="store_true", help="Cythonize the script." + ) diff --git a/pythonfmu/pythonfmu-export/src/pythonfmu/PySlaveInstance.cpp b/pythonfmu/pythonfmu-export/src/pythonfmu/PySlaveInstance.cpp index c0bd190..fce9eea 100644 --- a/pythonfmu/pythonfmu-export/src/pythonfmu/PySlaveInstance.cpp +++ b/pythonfmu/pythonfmu-export/src/pythonfmu/PySlaveInstance.cpp @@ -25,11 +25,69 @@ inline std::string getline(const std::string& fileName) return line; } + +std::string searchLeafClassName(PyObject* pLocals) +{ + std::string deepestFile = ""; + Py_ssize_t deepestChain = 0; + PyObject* key, * value; + Py_ssize_t pos = 0; + + while (PyDict_Next(pLocals, &pos, &key, &value)) { + // Check if element in namespace is a class + if (!PyType_Check(value)) { + continue; + } + + PyObject* pMroAttribute = PyObject_GetAttrString(value, "__mro__"); + + if (pMroAttribute != NULL && PySequence_Check(pMroAttribute)) { + std::regex pattern(" deepestChain && match[1] == "Fmi2Slave") { + deepestFile = PyBytes_AsString(PyUnicode_AsUTF8String(key)); + deepestChain = i; + } + } + } + Py_DECREF(pMroAttribute); + } + return deepestFile; +} + +PyObject* findClassFromBinary(const std::string& resources, const std::string& moduleName) { + + PyObject* pyModule = PyImport_ImportModule(moduleName.c_str()); + if (pyModule == nullptr) { + return nullptr; + } + + PyObject* pGlobals = PyModule_GetDict(pyModule); + auto deepestFile = searchLeafClassName(pGlobals); + + PyObject* pyClassName = Py_BuildValue("s", deepestFile.c_str()); + PyObject* pyClass = PyObject_GetAttr(pyModule, pyClassName); + + // Clean up Python objects + Py_DECREF(pyModule); + Py_DECREF(pyClassName); + return pyClass; + +} + PyObject* findClass(const std::string& resources, const std::string& moduleName) { // Initialize the Python interpreter std::string filename = resources + "/" + moduleName + ".py"; - std::string deepestFile = ""; - int deepestChain = 0; // Read and execute the Python file std::ifstream file; @@ -71,38 +129,7 @@ PyObject* findClass(const std::string& resources, const std::string& moduleName) } fileContents.clear(); - PyObject* key, * value; - Py_ssize_t pos = 0; - - while (PyDict_Next(pLocals, &pos, &key, &value)) { - // Check if element in namespace is a class - if (!PyType_Check(value)) { - continue; - } - - PyObject* pMroAttribute = PyObject_GetAttrString(value, "__mro__"); - - if (pMroAttribute != NULL && PySequence_Check(pMroAttribute)) { - std::regex pattern (" deepestChain && match[1] == "Fmi2Slave") { - deepestFile = PyBytes_AsString(PyUnicode_AsUTF8String(key)); - deepestChain = i; - } - } - } - Py_DECREF(pMroAttribute); - } + auto deepestFile = searchLeafClassName(pLocals); PyObject* pyClassName = Py_BuildValue("s", deepestFile.c_str()); PyObject* pyClass = PyObject_GetAttr(pyModule, pyClassName); @@ -150,8 +177,13 @@ PySlaveInstance::PySlaveInstance(std::string instanceName, std::string resources } std::string moduleName = getline(resources_ + "/slavemodule.txt"); - - pClass_ = findClass(resources_, moduleName); + std::string fileType = getline(resources_ + "/filetype.txt"); + + if (fileType == "bin") { + pClass_ = findClassFromBinary(resources_, moduleName); + } else { + pClass_ = findClass(resources_, moduleName); + } if (pClass_ == nullptr) { handle_py_exception("[ctor] findClass", gilState); } From 92815a5b368128b05922e0a050540b9637e8e1ab Mon Sep 17 00:00:00 2001 From: David Heejong Park Date: Fri, 11 Oct 2024 23:55:31 +0200 Subject: [PATCH 2/4] Dynamic loading of Cython --- pythonfmu/builder.py | 7 ++++++- requirements.txt | 5 +++-- setup.cfg | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pythonfmu/builder.py b/pythonfmu/builder.py index e0f7ed0..6f62649 100644 --- a/pythonfmu/builder.py +++ b/pythonfmu/builder.py @@ -16,7 +16,6 @@ from .osutil import get_lib_extension, get_platform from .fmi2slave import FMI2_MODEL_OPTIONS, Fmi2Slave from setuptools import setup -from Cython.Build import cythonize FilePath = Union[str, Path] HERE = Path(__file__).parent @@ -82,6 +81,12 @@ def build_FMU( **options, ) -> Path: has_cythonize: bool = options["cythonize"] + if has_cythonize: + cython_build_module = importlib.util.find_spec("Cython.Build") + cython = importlib.util.module_from_spec(cython_build_module) + cython_build_module.loader.exec_module(cython) + cythonize = cython.cythonize + script_file = Path(script_file) if not script_file.exists(): raise ValueError(f"No such file {script_file!s}") diff --git a/requirements.txt b/requirements.txt index c6cea94..d605ef5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -fmpy -pytest \ No newline at end of file +pytest~=8.3.3 +fmpy~=0.3.21 +cython~=3.0.11 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 3094b1f..20aa6a9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,8 +23,9 @@ classifiers = Programming Language :: Python :: 3.8 Topic :: Scientific/Engineering tests_require = - pytest - fmpy + pytest~=8.3.3 + fmpy~=0.3.21 + cython~=3.0.11 [options] include_package_data = True From 0a8bab1b6fe6219c7dbc1313aac4847530b527b8 Mon Sep 17 00:00:00 2001 From: David Heejong Park Date: Sat, 12 Oct 2024 00:26:12 +0200 Subject: [PATCH 3/4] Compiler directive --- pythonfmu/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonfmu/builder.py b/pythonfmu/builder.py index 6f62649..8c5dcc3 100644 --- a/pythonfmu/builder.py +++ b/pythonfmu/builder.py @@ -112,7 +112,7 @@ def build_FMU( if has_cythonize: setup( script_args=["build_ext", "--inplace"], - ext_modules=cythonize(str(script_file)), + ext_modules=cythonize(str(script_file), compiler_directives={"language_level": "3"}), ) for bin_file in glob.glob(f"{script_file.stem}*.{'pyd' if sys.platform == 'win32' else 'so'}"): shutil.copy2(bin_file, temp_dir) From e27e1c45fee6c94af85fd92e4137bf0e4b2dca95 Mon Sep 17 00:00:00 2001 From: David Heejong Park Date: Wed, 16 Oct 2024 07:35:27 +0200 Subject: [PATCH 4/4] Updated help and README --- README.md | 1 + pythonfmu/builder.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f5821ca..0a5aeef 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ optional arguments: -d DEST, --dest DEST Where to save the FMU. --doc DOCUMENTATION_FOLDER Documentation folder to include in the FMU. + -c, --cythonize Compile the Python script into a binary module using Cython. --no-external-tool If given, needsExecutionTool=false --no-variable-step If given, canHandleVariableCommunicationStepSize=false --interpolate-inputs If given, canInterpolateInputs=true diff --git a/pythonfmu/builder.py b/pythonfmu/builder.py index 8c5dcc3..c26c0ed 100644 --- a/pythonfmu/builder.py +++ b/pythonfmu/builder.py @@ -146,7 +146,7 @@ def build_FMU( shutil.copy2(file_, temp_dir) model_identifier, xml = get_model_description( - script_file if cythonize else temp_dir.absolute() / script_file.name, module_name + script_file if has_cythonize else temp_dir.absolute() / script_file.name, module_name ) dest_file = dest / f"{model_identifier}.fmu" @@ -170,7 +170,7 @@ def build_FMU( # Add information for the Python loader zip_fmu.writestr(str(resource.joinpath("slavemodule.txt")), module_name) - if cythonize: + if has_cythonize: zip_fmu.writestr(str(resource.joinpath("filetype.txt")), "bin") else: zip_fmu.writestr(str(resource.joinpath("filetype.txt")), "script")