diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c index ef910cab3d..2a3059f47b 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c @@ -31,6 +31,7 @@ #define ENTRYPOINT_MAXLEN 128 #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) #define LOGP(x) LOG("python", (x)) +#define P4A_MIN_VER 11 static PyObject *androidembed_log(PyObject *self, PyObject *args) { char *logstr = NULL; @@ -154,11 +155,6 @@ int main(int argc, char *argv[]) { Py_NoSiteFlag=1; #endif -#if PY_MAJOR_VERSION < 3 - Py_SetProgramName("android_python"); -#else - Py_SetProgramName(L"android_python"); -#endif #if PY_MAJOR_VERSION >= 3 /* our logging module for android @@ -174,40 +170,80 @@ int main(int argc, char *argv[]) { char python_bundle_dir[256]; snprintf(python_bundle_dir, 256, "%s/_python_bundle", getenv("ANDROID_UNPACK")); - if (dir_exists(python_bundle_dir)) { - LOGP("_python_bundle dir exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - python_bundle_dir, python_bundle_dir); - LOGP("calculated paths to be..."); - LOGP(paths); + #if PY_MAJOR_VERSION >= 3 - #if PY_MAJOR_VERSION >= 3 - wchar_t *wchar_paths = Py_DecodeLocale(paths, NULL); - Py_SetPath(wchar_paths); + #if PY_MINOR_VERSION >= P4A_MIN_VER + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.program_name = L"android_python"; + #else + Py_SetProgramName(L"android_python"); #endif - LOGP("set wchar paths..."); + #else + Py_SetProgramName("android_python"); + #endif + + if (dir_exists(python_bundle_dir)) { + LOGP("_python_bundle dir exists"); + + #if PY_MAJOR_VERSION >= 3 + #if PY_MINOR_VERSION >= P4A_MIN_VER + + wchar_t wchar_zip_path[256]; + wchar_t wchar_modules_path[256]; + swprintf(wchar_zip_path, 256, L"%s/stdlib.zip", python_bundle_dir); + swprintf(wchar_modules_path, 256, L"%s/modules", python_bundle_dir); + + config.module_search_paths_set = 1; + PyWideStringList_Append(&config.module_search_paths, wchar_zip_path); + PyWideStringList_Append(&config.module_search_paths, wchar_modules_path); + #else + char paths[512]; + snprintf(paths, 512, "%s/stdlib.zip:%s/modules", python_bundle_dir, python_bundle_dir); + wchar_t *wchar_paths = Py_DecodeLocale(paths, NULL); + Py_SetPath(wchar_paths); + #endif + + #endif + + LOGP("set wchar paths..."); } else { - LOGP("_python_bundle does not exist...this not looks good, all python" - " recipes should have this folder, should we expect a crash soon?"); + LOGP("_python_bundle does not exist...this not looks good, all python" + " recipes should have this folder, should we expect a crash soon?"); } - Py_Initialize(); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= P4A_MIN_VER + PyStatus status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + LOGP("Python initialization failed:"); + LOGP(status.err_msg); + } +#else + Py_Initialize(); + LOGP("Python initialized using legacy Py_Initialize()."); +#endif + LOGP("Initialized python"); - /* ensure threads will work. - */ - LOGP("AND: Init threads"); - PyEval_InitThreads(); + /* < 3.9 requires explicit GIL initialization + * 3.9+ PyEval_InitThreads() is deprecated and unnecessary + */ + #if PY_VERSION_HEX < 0x03090000 + LOGP("Initializing threads (required for Python < 3.9)"); + PyEval_InitThreads(); + #endif #if PY_MAJOR_VERSION < 3 initandroidembed(); #endif - PyRun_SimpleString("import androidembed\nandroidembed.log('testing python " - "print redirection')"); + PyRun_SimpleString( + "import androidembed\n" + "androidembed.log('testing python print redirection')" + + ); /* inject our bootstrap code to redirect python stdin/stdout * replace sys.path with our path @@ -325,20 +361,6 @@ int main(int argc, char *argv[]) { LOGP("Python for android ended."); - /* Shut down: since regular shutdown causes issues sometimes - (seems to be an incomplete shutdown breaking next launch) - we'll use sys.exit(ret) to shutdown, since that one works. - - Reference discussion: - - https://github.com/kivy/kivy/pull/6107#issue-246120816 - */ - char terminatecmd[256]; - snprintf( - terminatecmd, sizeof(terminatecmd), - "import sys; sys.exit(%d)\n", ret - ); - PyRun_SimpleString(terminatecmd); /* This should never actually be reached, but we'll leave the clean-up * here just to be safe. diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index 83d11639bb..711d809f4f 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -39,27 +39,22 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt } protected static ArrayList getLibraries(File libsDir) { - ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "sqlite3", libsDir); - addLibraryIfExists(libsList, "ffi", libsDir); - addLibraryIfExists(libsList, "png16", libsDir); - addLibraryIfExists(libsList, "ssl.*", libsDir); - addLibraryIfExists(libsList, "crypto.*", libsDir); - addLibraryIfExists(libsList, "SDL2", libsDir); - addLibraryIfExists(libsList, "SDL2_image", libsDir); - addLibraryIfExists(libsList, "SDL2_mixer", libsDir); - addLibraryIfExists(libsList, "SDL2_ttf", libsDir); - addLibraryIfExists(libsList, "SDL3", libsDir); - addLibraryIfExists(libsList, "SDL3_image", libsDir); - addLibraryIfExists(libsList, "SDL3_mixer", libsDir); - addLibraryIfExists(libsList, "SDL3_ttf", libsDir); - libsList.add("python3.5m"); - libsList.add("python3.6m"); - libsList.add("python3.7m"); - libsList.add("python3.8"); - libsList.add("python3.9"); - libsList.add("python3.10"); - libsList.add("python3.11"); + ArrayList libsList = new ArrayList<>(); + + String[] libNames = { + "sqlite3", "ffi", "png16", "ssl.*", "crypto.*", + "SDL2", "SDL2_image", "SDL2_mixer", "SDL2_ttf", + "SDL3", "SDL3_image", "SDL3_mixer", "SDL3_ttf" + }; + + for (String name : libNames) { + addLibraryIfExists(libsList, name, libsDir); + } + + for (int v = 5; v <= 13; v++) { + libsList.add("python3." + v + (v <= 7 ? "m" : "")); + } + libsList.add("main"); return libsList; } @@ -79,7 +74,7 @@ public static void loadLibraries(File filesDir, File libsDir) { // load, and it has failed, give a more // general error Log.v(TAG, "Library loading error: " + e.getMessage()); - if (lib.startsWith("python3.11") && !foundPython) { + if (lib.startsWith("python3.13") && !foundPython) { throw new RuntimeException("Could not load any libpythonXXX.so"); } else if (lib.startsWith("python")) { continue; diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 0ebe005d14..4897891961 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -12,6 +12,7 @@ from urllib.request import urlretrieve from os import listdir, unlink, environ, curdir, walk from sys import stdout +from multiprocessing import cpu_count import time try: from urlparse import urlparse @@ -517,7 +518,7 @@ def unpack(self, arch): for entry in listdir(extraction_filename): # Previously we filtered out the .git folder, but during the build process for some recipes # (e.g. when version is parsed by `setuptools_scm`) that may be needed. - shprint(sh.cp, '-Rv', + shprint(sh.cp, '-R', join(extraction_filename, entry), directory_name) else: @@ -830,6 +831,8 @@ def build_arch(self, arch, *extra_args): shprint( sh.Command(join(self.ctx.ndk_dir, "ndk-build")), 'V=1', + "-j", + str(cpu_count()), 'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"), 'APP_PLATFORM=android-' + str(self.ctx.ndk_api), 'APP_ABI=' + arch.arch, @@ -878,6 +881,8 @@ class PythonRecipe(Recipe): hostpython_prerequisites = [] '''List of hostpython packages required to build a recipe''' + _host_recipe = None + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if 'python3' not in self.depends: @@ -890,6 +895,10 @@ def __init__(self, *args, **kwargs): depends = list(set(depends)) self.depends = depends + def prebuild_arch(self, arch): + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + return super().prebuild_arch(arch) + def clean_build(self, arch=None): super().clean_build(arch=arch) name = self.folder_name @@ -907,8 +916,7 @@ def clean_build(self, arch=None): def real_hostpython_location(self): host_name = 'host{}'.format(self.ctx.python_recipe.name) if host_name == 'hostpython3': - python_recipe = Recipe.get_recipe(host_name, self.ctx) - return python_recipe.python_exe + return self._host_recipe.python_exe else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -927,14 +935,44 @@ def folder_name(self): name = self.name return name + def patch_shebang(self, _file, original_bin): + _file_des = open(_file, "r") + + try: + data = _file_des.readlines() + except UnicodeDecodeError: + return + + if "#!" in (line := data[0]): + if line.split("#!")[-1].strip() == original_bin: + return + + info(f"Fixing shebang for '{_file}'") + data.pop(0) + data.insert(0, "#!" + original_bin + "\n") + _file_des.close() + _file_des = open(_file, "w") + _file_des.write("".join(data)) + _file_des.close() + + def patch_shebangs(self, path, original_bin): + # set correct shebang + for file in listdir(path): + _file = join(path, file) + self.patch_shebang(_file, original_bin) + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + if self._host_recipe is None: + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + env = super().get_recipe_env(arch, with_flags_in_cc) - env['PYTHONNOUSERSITE'] = '1' # Set the LANG, this isn't usually important but is a better default # as it occasionally matters how Python e.g. reads files env['LANG'] = "en_GB.UTF-8" # Binaries made by packages installed by pip - env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"] + env["PATH"] = self._host_recipe.site_bin + ":" + env["PATH"] + host_env = self.get_hostrecipe_env() + env['PYTHONPATH'] = host_env["PYTHONPATH"] if not self.call_hostpython_via_targetpython: env['CFLAGS'] += ' -I{}'.format( @@ -945,18 +983,6 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): self.ctx.python_recipe.link_version, ) - hppath = [] - hppath.append(join(dirname(self.hostpython_location), 'Lib')) - hppath.append(join(hppath[0], 'site-packages')) - builddir = join(dirname(self.hostpython_location), 'build') - if exists(builddir): - hppath += [join(builddir, d) for d in listdir(builddir) - if isdir(join(builddir, d))] - if len(hppath) > 0: - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']]) - else: - env['PYTHONPATH'] = ':'.join(hppath) return env def should_build(self, arch): @@ -993,13 +1019,23 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): '--install-lib=.', _env=hpenv, *self.setup_extra_args) - # If asked, also install in the hostpython build dir - if self.install_in_hostpython: - self.install_hostpython_package(arch) + if isfile("setup.py"): + shprint(hostpython, 'setup.py', 'install', '-O2', + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), + '--install-lib=.', + _env=hpenv, *self.setup_extra_args) + + # If asked, also install in the hostpython build dir + if self.install_in_hostpython: + self.install_hostpython_package(arch) + else: + warning("`PythonRecipe.install_python_package` called without `setup.py` file!") - def get_hostrecipe_env(self, arch): + def get_hostrecipe_env(self): env = environ.copy() - env['PYTHONPATH'] = self.hostpython_site_dir + _python_path = self._host_recipe.get_path_to_python() + env['PYTHONPATH'] = self._host_recipe.site_dir + ":" + join( + _python_path, "Modules") + ":" + glob.glob(join(_python_path, "build", "lib*"))[0] return env @property @@ -1007,11 +1043,12 @@ def hostpython_site_dir(self): return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages') def install_hostpython_package(self, arch): - env = self.get_hostrecipe_env(arch) + env = self.get_hostrecipe_env() real_hostpython = sh.Command(self.real_hostpython_location) shprint(real_hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(dirname(self.real_hostpython_location)), '--install-lib=Lib/site-packages', + '--root={}'.format(self._host_recipe.site_root), _env=env, *self.setup_extra_args) @property @@ -1029,7 +1066,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): pip_options = [ "install", *packages, - "--target", self.hostpython_site_dir, "--python-version", + "--target", self._host_recipe.site_dir, "--python-version", self.ctx.python_recipe.version, # Don't use sources, instead wheels "--only-binary=:all:", @@ -1037,7 +1074,9 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): if force_upgrade: pip_options.append("--upgrade") # Use system's pip - shprint(sh.pip, *pip_options) + pip_env = self.get_hostrecipe_env() + pip_env["HOME"] = "/tmp" + shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env) def restore_hostpython_prerequisites(self, packages): _packages = [] @@ -1076,7 +1115,7 @@ def build_compiled_components(self, arch): env['STRIP'], '{}', ';', _env=env) def install_hostpython_package(self, arch): - env = self.get_hostrecipe_env(arch) + env = self.get_hostrecipe_env() self.rebuild_compiled_components(arch, env) super().install_hostpython_package(arch) @@ -1231,7 +1270,7 @@ def get_recipe_env(self, arch, **kwargs): return env def get_wheel_platform_tag(self, arch): - return "android_" + { + return f"android_{self.ctx.ndk_api}_" + { "armeabi-v7a": "arm", "arm64-v8a": "aarch64", "x86_64": "x86_64", @@ -1268,6 +1307,9 @@ def build_arch(self, arch): self.install_hostpython_prerequisites( packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites ) + python_bin_dir = join(self._host_recipe.site_dir, "bin") + self.patch_shebangs(python_bin_dir, self.real_hostpython_location) + build_dir = self.get_build_dir(arch.arch) env = self.get_recipe_env(arch, with_flags_in_cc=True) # make build dir separately diff --git a/pythonforandroid/recipes/android/__init__.py b/pythonforandroid/recipes/android/__init__.py index 5174a69bfa..9c8bddd168 100644 --- a/pythonforandroid/recipes/android/__init__.py +++ b/pythonforandroid/recipes/android/__init__.py @@ -1,11 +1,11 @@ -from pythonforandroid.recipe import CythonRecipe, IncludedFilesBehaviour +from pythonforandroid.recipe import PyProjectRecipe, IncludedFilesBehaviour from pythonforandroid.util import current_directory from pythonforandroid import logger from os.path import join -class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): +class AndroidRecipe(IncludedFilesBehaviour, PyProjectRecipe): # name = 'android' version = None url = None @@ -13,11 +13,12 @@ class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): src_filename = 'src' depends = [('sdl3', 'sdl2', 'genericndkbuild'), 'pyjnius'] + hostpython_prerequisites = ["cython"] config_env = {} - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) env.update(self.config_env) return env @@ -49,6 +50,7 @@ def prebuild_arch(self, arch): 'IS_SDL2': int(bootstrap_name == "sdl2"), 'IS_SDL3': int(bootstrap_name == "sdl3"), 'PY2': 0, + 'ANDROID_LIBS_DIR': self.ctx.get_libs_dir(arch.arch), 'JAVA_NAMESPACE': java_ns, 'JNI_NAMESPACE': jni_ns, 'ACTIVITY_CLASS_NAME': self.ctx.activity_class_name, diff --git a/pythonforandroid/recipes/android/src/setup.py b/pythonforandroid/recipes/android/src/setup.py index 0f5ceb1fd3..a329a9de8b 100755 --- a/pythonforandroid/recipes/android/src/setup.py +++ b/pythonforandroid/recipes/android/src/setup.py @@ -1,25 +1,34 @@ from distutils.core import setup, Extension +from Cython.Build import cythonize import os -library_dirs = ['libs/' + os.environ['ARCH']] +library_dirs = [os.environ['ANDROID_LIBS_DIR']] lib_dict = { 'sdl2': ['SDL2', 'SDL2_image', 'SDL2_mixer', 'SDL2_ttf'], 'sdl3': ['SDL3', 'SDL3_image', 'SDL3_mixer', 'SDL3_ttf'], } sdl_libs = lib_dict.get(os.environ['BOOTSTRAP'], ['main']) -modules = [Extension('android._android', - ['android/_android.c', 'android/_android_jni.c'], - libraries=sdl_libs + ['log'], - library_dirs=library_dirs), - Extension('android._android_billing', - ['android/_android_billing.c', 'android/_android_billing_jni.c'], - libraries=['log'], - library_dirs=library_dirs)] +modules = [ + Extension('android._android', + ['android/_android.pyx', 'android/_android_jni.c'], + libraries=sdl_libs + ['log'], + library_dirs=library_dirs), + Extension('android._android_billing', + ['android/_android_billing.pyx', 'android/_android_billing_jni.c'], + libraries=['log'], + library_dirs=library_dirs), + Extension('android._android_sound', + ['android/_android_sound.pyx', 'android/_android_sound_jni.c'], + libraries=['log'], + library_dirs=library_dirs) +] + +cythonized_modules = cythonize(modules, compiler_directives={'language_level': "3"}) setup(name='android', version='1.0', packages=['android'], package_dir={'android': 'android'}, - ext_modules=modules + ext_modules=cythonized_modules ) diff --git a/pythonforandroid/recipes/genericndkbuild/__init__.py b/pythonforandroid/recipes/genericndkbuild/__init__.py index 9e85aac5d6..1b071ffc84 100644 --- a/pythonforandroid/recipes/genericndkbuild/__init__.py +++ b/pythonforandroid/recipes/genericndkbuild/__init__.py @@ -1,5 +1,5 @@ from os.path import join - +from multiprocessing import cpu_count from pythonforandroid.recipe import BootstrapNDKRecipe from pythonforandroid.toolchain import current_directory, shprint import sh @@ -29,7 +29,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_jni_dir()): - shprint(sh.Command(join(self.ctx.ndk_dir, "ndk-build")), "V=1", _env=env) + shprint(sh.Command(join(self.ctx.ndk_dir, "ndk-build")), "V=1", "-j", str(cpu_count()), _env=env) recipe = GenericNDKBuildRecipe() diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index 9ba4580019..45c5091990 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -5,6 +5,7 @@ from pathlib import Path from os.path import join +from packaging.version import Version from pythonforandroid.logger import shprint from pythonforandroid.recipe import Recipe from pythonforandroid.util import ( @@ -35,17 +36,16 @@ class HostPython3Recipe(Recipe): :class:`~pythonforandroid.python.HostPythonRecipe` ''' - version = '3.11.5' - name = 'hostpython3' + version = '3.13.5' + _p_version = Version(version) + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' build_subdir = 'native-build' '''Specify the sub build directory for the hostpython3 recipe. Defaults to ``native-build``.''' - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' '''The default url to download our host python recipe. This url will change depending on the python version set in attribute :attr:`version`.''' - patches = ['patches/pyconfig_detection.patch'] @property @@ -95,6 +95,26 @@ def get_build_dir(self, arch=None): def get_path_to_python(self): return join(self.get_build_dir(), self.build_subdir) + @property + def site_root(self): + return join(self.get_path_to_python(), "root") + + @property + def site_bin(self): + dir = None + # TODO: implement mac os + # if os.name == "posix": + dir = "usr/local/bin/" + return join(self.site_root, dir) + + @property + def site_dir(self): + dir = None + # TODO: implement mac os + # if os.name == "posix": + dir = f"usr/local/lib/python{self._p_version.major}.{self._p_version.minor}/site-packages/" + return join(self.site_root, dir) + def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -138,7 +158,12 @@ def build_arch(self, arch): shprint(sh.cp, exe, self.python_exe) break + ensure_dir(self.site_root) self.ctx.hostpython = self.python_exe + shprint( + sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U", + _env={"HOME": "/tmp"} + ) recipe = HostPython3Recipe() diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index f80024155f..13ff577676 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -25,7 +25,7 @@ class KivyRecipe(PyProjectRecipe): url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' - depends = [('sdl2', 'sdl3'), 'pyjnius', 'setuptools'] + depends = [('sdl2', 'sdl3'), 'pyjnius', 'setuptools', 'android'] python_depends = ['certifi', 'chardet', 'idna', 'requests', 'urllib3', 'filetype'] hostpython_prerequisites = [] diff --git a/pythonforandroid/recipes/libcurl/__init__.py b/pythonforandroid/recipes/libcurl/__init__.py index 2971532fb1..1804a243af 100644 --- a/pythonforandroid/recipes/libcurl/__init__.py +++ b/pythonforandroid/recipes/libcurl/__init__.py @@ -7,11 +7,15 @@ class LibcurlRecipe(Recipe): - version = '7.55.1' - url = 'https://curl.haxx.se/download/curl-7.55.1.tar.gz' + version = '8.8.0' + url = 'https://github.com/curl/curl/releases/download/curl-{_version}/curl-{version}.tar.gz' built_libraries = {'libcurl.so': 'dist/lib'} depends = ['openssl'] + @property + def versioned_url(self): + return self.url.format(version=self.version, _version=self.version.replace(".", "_")) + def build_arch(self, arch): env = self.get_recipe_env(arch) diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 767881b793..d63c173f4b 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -14,8 +14,8 @@ class LibffiRecipe(Recipe): - `libltdl-dev` which defines the `LT_SYS_SYMBOL_USCORE` macro """ name = 'libffi' - version = 'v3.4.2' - url = 'https://github.com/libffi/libffi/archive/{version}.tar.gz' + version = '3.4.6' + url = 'https://github.com/libffi/libffi/archive/v{version}.tar.gz' patches = ['remove-version-info.patch'] diff --git a/pythonforandroid/recipes/liblzma/__init__.py b/pythonforandroid/recipes/liblzma/__init__.py index 0b880bc484..8144112322 100644 --- a/pythonforandroid/recipes/liblzma/__init__.py +++ b/pythonforandroid/recipes/liblzma/__init__.py @@ -11,7 +11,7 @@ class LibLzmaRecipe(Recipe): - version = '5.2.4' + version = '5.6.2' url = 'https://tukaani.org/xz/xz-{version}.tar.gz' built_libraries = {'liblzma.so': 'p4a_install/lib'} diff --git a/pythonforandroid/recipes/libsodium/__init__.py b/pythonforandroid/recipes/libsodium/__init__.py index a8a1909588..f66fc18e7f 100644 --- a/pythonforandroid/recipes/libsodium/__init__.py +++ b/pythonforandroid/recipes/libsodium/__init__.py @@ -3,24 +3,15 @@ from pythonforandroid.logger import shprint from multiprocessing import cpu_count import sh -from packaging import version as packaging_version class LibsodiumRecipe(Recipe): version = '1.0.16' - url = 'https://github.com/jedisct1/libsodium/releases/download/{}/libsodium-{}.tar.gz' + url = 'https://github.com/jedisct1/libsodium/releases/download/{version}/libsodium-{version}.tar.gz' depends = [] patches = ['size_max_fix.patch'] built_libraries = {'libsodium.so': 'src/libsodium/.libs'} - @property - def versioned_url(self): - asked_version = packaging_version.parse(self.version) - if asked_version > packaging_version.parse('1.0.16'): - return self._url.format(self.version + '-RELEASE', self.version) - else: - return self._url.format(self.version, self.version) - def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 766c10e361..9a9a8c8a0f 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,4 +1,5 @@ from os.path import join +from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.util import current_directory @@ -44,35 +45,23 @@ class OpenSSLRecipe(Recipe): ''' - version = '1.1' - '''the major minor version used to link our recipes''' - - url_version = '1.1.1w' - '''the version used to download our libraries''' - - url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz' + version = '3.3.1' + url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' built_libraries = { - 'libcrypto{version}.so'.format(version=version): '.', - 'libssl{version}.so'.format(version=version): '.', + 'libcrypto.so': '.', + 'libssl.so': '.', } - @property - def versioned_url(self): - if self.url is None: - return None - return self.url.format(url_version=self.url_version) - def get_build_dir(self, arch): return join( - self.get_build_container_dir(arch), self.name + self.version + self.get_build_container_dir(arch), self.name + self.version[0] ) def include_flags(self, arch): '''Returns a string with the include folders''' openssl_includes = join(self.get_build_dir(arch.arch), 'include') return (' -I' + openssl_includes + - ' -I' + join(openssl_includes, 'internal') + ' -I' + join(openssl_includes, 'openssl')) def link_dirs_flags(self, arch): @@ -85,7 +74,7 @@ def link_libs_flags(self): '''Returns a string with the appropriate `-l` flags to link with the openssl libs. This string is usually added to the environment variable `LIBS`''' - return ' -lcrypto{version} -lssl{version}'.format(version=self.version) + return ' -lcrypto -lssl' def link_flags(self, arch): '''Returns a string with the flags to link with the openssl libraries @@ -94,10 +83,12 @@ def link_flags(self, arch): def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) - env['OPENSSL_VERSION'] = self.version - env['MAKE'] = 'make' # This removes the '-j5', which isn't safe + env['OPENSSL_VERSION'] = self.version[0] env['CC'] = 'clang' - env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir + env['ANDROID_NDK_ROOT'] = self.ctx.ndk_dir + env["PATH"] = f"{self.ctx.ndk.llvm_bin_dir}:{env['PATH']}" + env["CFLAGS"] += " -Wno-macro-redefined" + env["MAKE"] = "make" return env def select_build_arch(self, arch): @@ -125,13 +116,12 @@ def build_arch(self, arch): 'shared', 'no-dso', 'no-asm', + 'no-tests', buildarch, '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), ] shprint(perl, 'Configure', *config_args, _env=env) - self.apply_patch('disable-sover.patch', arch.arch) - - shprint(sh.make, 'build_libs', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) recipe = OpenSSLRecipe() diff --git a/pythonforandroid/recipes/openssl/disable-sover.patch b/pythonforandroid/recipes/openssl/disable-sover.patch deleted file mode 100644 index d944483cda..0000000000 --- a/pythonforandroid/recipes/openssl/disable-sover.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- openssl/Makefile.orig 2018-10-20 22:49:40.418310423 +0200 -+++ openssl/Makefile 2018-10-20 22:50:23.347322403 +0200 -@@ -19,7 +19,7 @@ - SHLIB_MAJOR=1 - SHLIB_MINOR=1 - SHLIB_TARGET=linux-shared --SHLIB_EXT=.so.$(SHLIB_VERSION_NUMBER) -+SHLIB_EXT=$(SHLIB_VERSION_NUMBER).so - SHLIB_EXT_SIMPLE=.so - SHLIB_EXT_IMPORT= - diff --git a/pythonforandroid/recipes/pyjnius/__init__.py b/pythonforandroid/recipes/pyjnius/__init__.py index c6b6746fa1..15c6e74d86 100644 --- a/pythonforandroid/recipes/pyjnius/__init__.py +++ b/pythonforandroid/recipes/pyjnius/__init__.py @@ -7,14 +7,13 @@ class PyjniusRecipe(PyProjectRecipe): version = '1.6.1' - url = 'https://github.com/kivy/pyjnius/archive/{version}.zip' + url = 'https://github.com/cmacdonald/pyjnius/archive/refs/heads/long_redux.zip' name = 'pyjnius' depends = [('genericndkbuild', 'sdl2', 'sdl3'), 'six'] site_packages_name = 'jnius' patches = [ "use_cython.patch", - "cython_version_pin.patch", ('genericndkbuild_jnienv_getter.patch', will_build('genericndkbuild')), ('sdl3_jnienv_getter.patch', will_build('sdl3')), ] diff --git a/pythonforandroid/recipes/pyjnius/cython_version_pin.patch b/pythonforandroid/recipes/pyjnius/cython_version_pin.patch deleted file mode 100644 index 3d5cea2350..0000000000 --- a/pythonforandroid/recipes/pyjnius/cython_version_pin.patch +++ /dev/null @@ -1,9 +0,0 @@ ---- pyjnius-1.6.1/pyproject.toml 2023-11-05 21:07:43.000000000 +0530 -+++ pyjnius-1.6.1.mod/pyproject.toml 2025-05-11 20:31:14.699072764 +0530 -@@ -2,5 +2,5 @@ - requires = [ - "setuptools>=58.0.0", - "wheel", -- "Cython" -+ "Cython==3.0.0" - ] diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 7cd3928d76..88a8ebf7b7 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -3,12 +3,12 @@ import subprocess from os import environ, utime -from os.path import dirname, exists, join -from pathlib import Path +from os.path import dirname, exists, join, isfile +from multiprocessing import cpu_count import shutil -from pythonforandroid.logger import info, warning, shprint -from pythonforandroid.patching import version_starts_with +from packaging.version import Version +from pythonforandroid.logger import info, shprint from pythonforandroid.recipe import Recipe, TargetPythonRecipe from pythonforandroid.util import ( current_directory, @@ -55,34 +55,34 @@ class Python3Recipe(TargetPythonRecipe): :class:`~pythonforandroid.python.GuestPythonRecipe` ''' - version = '3.11.5' - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' + version = '3.13.5' + _p_version = Version(version) + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' name = 'python3' patches = [ 'patches/pyconfig_detection.patch', 'patches/reproducible-buildinfo.diff', - - # Python 3.7.1 - ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")), - ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")), - - # Python 3.8.1 & 3.9.X - ('patches/py3.8.1.patch', version_starts_with("3.8")), - ('patches/py3.8.1.patch', version_starts_with("3.9")), - ('patches/py3.8.1.patch', version_starts_with("3.10")), - ('patches/cpython-311-ctypes-find-library.patch', version_starts_with("3.11")), ] - if shutil.which('lld') is not None: + if _p_version.major == 3 and _p_version.minor == 7: patches += [ - ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.11")), + 'patches/py3.7.1_fix-ctypes-util-find-library.patch', + 'patches/py3.7.1_fix-zlib-version.patch', ] + if _p_version.minor in (8, 9, 10): + patches.append('patches/py3.8.1.patch') + + if _p_version.minor == 11: + patches.append('patches/cpython-311-ctypes-find-library.patch') + + if shutil.which('lld') is not None: + if _p_version.minor == 7: + patches.append("patches/py3.7.1_fix_cortex_a8.patch") + elif _p_version.minor in (8, 9, 10, 11): + patches.append("patches/py3.8.1_fix_cortex_a8.patch") + depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] # those optional depends allow us to build python compression modules: # - _bz2.so @@ -90,23 +90,33 @@ class Python3Recipe(TargetPythonRecipe): opt_depends = ['libbz2', 'liblzma'] '''The optional libraries which we would like to get our python linked''' - configure_args = ( + configure_args = [ '--host={android_host}', '--build={android_build}', '--enable-shared', '--enable-ipv6', - 'ac_cv_file__dev_ptmx=yes', - 'ac_cv_file__dev_ptc=no', + '--enable-loadable-sqlite-extensions', + '--without-static-libpython', + '--without-readline', '--without-ensurepip', - 'ac_cv_little_endian_double=yes', - 'ac_cv_header_sys_eventfd_h=no', + + # Android prefix '--prefix={prefix}', '--exec-prefix={exec_prefix}', '--enable-loadable-sqlite-extensions' - ) - if version_starts_with("3.11"): - configure_args += ('--with-build-python={python_host_bin}',) + # Special cross compile args + 'ac_cv_file__dev_ptmx=yes', + 'ac_cv_file__dev_ptc=no', + 'ac_cv_header_sys_eventfd_h=no', + 'ac_cv_little_endian_double=yes', + 'ac_cv_header_bzlib_h=no', + ] + + if _p_version.minor >= 11: + configure_args.extend([ + '--with-build-python={python_host_bin}', + ]) '''The configure arguments needed to build the python recipe. Those are used in method :meth:`build_arch` (if not overwritten like python3's @@ -168,6 +178,9 @@ class Python3Recipe(TargetPythonRecipe): longer used and has been removed in favour of extension .pyc ''' + disable_gil = False + '''python3.13 experimental free-threading build''' + def __init__(self, *args, **kwargs): self._ctx = None super().__init__(*args, **kwargs) @@ -199,7 +212,7 @@ def link_root(self, arch_name): return join(self.get_build_dir(arch_name), 'android-build') def should_build(self, arch): - return not Path(self.link_root(arch.arch), self._libpython).is_file() + return not isfile(join(self.link_root(arch.arch), self._libpython)) def prebuild_arch(self, arch): super().prebuild_arch(arch) @@ -225,13 +238,10 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): ) env['LDFLAGS'] = env.get('LDFLAGS', '') - if shutil.which('lld') is not None: - # Note: The -L. is to fix a bug in python 3.7. - # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 - env['LDFLAGS'] += ' -L. -fuse-ld=lld' - else: - warning('lld not found, linking without it. ' - 'Consider installing lld if linker errors occur.') + # Note: The -L. is to fix a bug in python 3.7. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 + env["PATH"] = f"{self.ctx.ndk.llvm_bin_dir}:{env['PATH']}" # find lld + env['LDFLAGS'] += ' -L. -fuse-ld=lld' return env @@ -244,30 +254,26 @@ def add_flags(include_flags, link_dirs, link_libs): env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs env['LIBS'] = env.get('LIBS', '') + link_libs - if 'sqlite3' in self.ctx.recipe_build_order: - info('Activating flags for sqlite3') - recipe = Recipe.get_recipe('sqlite3', self.ctx) - add_flags(' -I' + recipe.get_build_dir(arch.arch), - ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') - - if 'libffi' in self.ctx.recipe_build_order: - info('Activating flags for libffi') - recipe = Recipe.get_recipe('libffi', self.ctx) - # In order to force the correct linkage for our libffi library, we - # set the following variable to point where is our libffi.pc file, - # because the python build system uses pkg-config to configure it. - env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) - add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), - ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), - ' -lffi') - - if 'openssl' in self.ctx.recipe_build_order: - info('Activating flags for openssl') - recipe = Recipe.get_recipe('openssl', self.ctx) - self.configure_args += \ - ('--with-openssl=' + recipe.get_build_dir(arch.arch),) - add_flags(recipe.include_flags(arch), - recipe.link_dirs_flags(arch), recipe.link_libs_flags()) + info('Activating flags for sqlite3') + recipe = Recipe.get_recipe('sqlite3', self.ctx) + add_flags(' -I' + recipe.get_build_dir(arch.arch), + ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') + + info('Activating flags for libffi') + recipe = Recipe.get_recipe('libffi', self.ctx) + # In order to force the correct linkage for our libffi library, we + # set the following variable to point where is our libffi.pc file, + # because the python build system uses pkg-config to configure it. + env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch) + add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), + ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), + ' -lffi') + + info('Activating flags for openssl') + recipe = Recipe.get_recipe('openssl', self.ctx) + self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch)) + add_flags(recipe.include_flags(arch), + recipe.link_dirs_flags(arch), recipe.link_libs_flags()) for library_name in {'libbz2', 'liblzma'}: if library_name in self.ctx.recipe_build_order: @@ -303,6 +309,9 @@ def add_flags(include_flags, link_dirs, link_libs): env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') + if self._p_version.minor >= 13 and self.disable_gil: + self.configure_args.append("--disable-gil") + return env def build_arch(self, arch): @@ -330,6 +339,9 @@ def build_arch(self, arch): join(recipe_build_dir, 'config.guess'))().strip() + # disable blake2 fix + self.configure_args.append("--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3") + with current_directory(build_dir): if not exists('config.status'): shprint( @@ -344,11 +356,10 @@ def build_arch(self, arch): exec_prefix=sys_exec_prefix)).split(' '), _env=env) - # Python build does not seem to play well with make -j option from Python 3.11 and onwards - # Before losing some time, please check issue - # https://github.com/python/cpython/issues/101295 , as the root cause looks similar shprint( sh.make, + '-j', + str(cpu_count()), 'all', 'INSTSONAME={lib_name}'.format(lib_name=self._libpython), _env=env @@ -382,11 +393,13 @@ def create_python_bundle(self, dirn, arch): self.get_build_dir(arch.arch), 'android-build', 'build', - 'lib.linux{}-{}-{}'.format( + 'lib.{}{}-{}-{}'.format( + # android is now supported platform + "android" if self._p_version.minor >= 13 else "linux", '2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string - )) + )) # Compile to *.pyc the python modules self.compile_python_files(modules_build_dir) diff --git a/pythonforandroid/recipes/sdl2/__init__.py b/pythonforandroid/recipes/sdl2/__init__.py index d1a5fdc8b3..20f5d54a81 100644 --- a/pythonforandroid/recipes/sdl2/__init__.py +++ b/pythonforandroid/recipes/sdl2/__init__.py @@ -1,8 +1,10 @@ +from multiprocessing import cpu_count from os.path import exists, join +import sh + from pythonforandroid.recipe import BootstrapNDKRecipe from pythonforandroid.toolchain import current_directory, shprint -import sh class LibSDL2Recipe(BootstrapNDKRecipe): @@ -14,7 +16,7 @@ class LibSDL2Recipe(BootstrapNDKRecipe): dir_name = 'SDL' - depends = ['sdl2_image', 'sdl2_mixer', 'sdl2_ttf'] + depends = ['sdl2_image', 'sdl2_mixer', 'sdl2_ttf', 'python3'] def get_recipe_env(self, arch=None, with_flags_in_cc=True, with_python=True): env = super().get_recipe_env( @@ -34,6 +36,8 @@ def build_arch(self, arch): shprint( sh.Command(join(self.ctx.ndk_dir, "ndk-build")), "V=1", + "-j", + str(cpu_count()), "NDK_DEBUG=" + ("1" if self.ctx.build_as_debuggable else "0"), _env=env ) diff --git a/pythonforandroid/recipes/sdl3/__init__.py b/pythonforandroid/recipes/sdl3/__init__.py index 6b9c8ee7c6..745ba38aed 100644 --- a/pythonforandroid/recipes/sdl3/__init__.py +++ b/pythonforandroid/recipes/sdl3/__init__.py @@ -14,7 +14,7 @@ class LibSDL3Recipe(BootstrapNDKRecipe): dir_name = "SDL" - depends = ["sdl3_image", "sdl3_mixer", "sdl3_ttf"] + depends = ["sdl3_image", "sdl3_mixer", "sdl3_ttf", "python3"] def get_recipe_env( self, arch=None, with_flags_in_cc=True, with_python=True diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py deleted file mode 100644 index 3be8ce7578..0000000000 --- a/pythonforandroid/recipes/six/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class SixRecipe(PythonRecipe): - version = '1.15.0' - url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' - depends = ['setuptools'] - - -recipe = SixRecipe()