Update Android test-command handling (#2590)

* Update Android test-command handling

* Update to Python 3.13.8

* Update documentation and tests

* Deal with `sysconfig.get_config_var("exec_prefix")` changing in Python 3.14, and add cross venv tests

* Allow test commands starting with `python3`

* test-command cleanups

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: mhsmith

README.md

@@ -70,7 +70,7 @@ Usage
 <sup>² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.</sup><br>
 <sup>³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. </sup><br>
 <sup>⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).</sup><br>
-<sup>⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](platforms/#ios-system-requirements)</sup><br> when building iOS wheels.
+<sup>⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](https://cibuildwheel.pypa.io/en/stable/platforms/#ios-system-requirements) when building iOS wheels.</sup><br>
 
 <!--intro-end-->
 

cibuildwheel/platforms/android.py

@@ -617,8 +617,11 @@ def test_wheel(state: BuildState, wheel: Path) -> None:
 
     # Parse test-command.
     test_args = shlex.split(test_command)
-    if test_args[:2] in [["python", "-c"], ["python", "-m"]]:
-        test_args[:3] = [test_args[1], test_args[2], "--"]
+    if test_args[0] in ["python", "python3"] and any(arg in test_args for arg in ["-c", "-m"]):
+        # Forward the args to the CPython testbed script. We require '-c' or '-m'
+        # to be in the command, because without those flags, the testbed script
+        # will prepend '-m test', which will run Python's own test suite.
+        del test_args[0]
     elif test_args[0] in ["pytest"]:
         # We transform some commands into the `python -m` form, but this is deprecated.
         msg = (
@@ -627,11 +630,11 @@ def test_wheel(state: BuildState, wheel: Path) -> None:
             "If this works, all you need to do is add that to your test command."
         )
         log.warning(msg)
-        test_args[:1] = ["-m", test_args[0], "--"]
+        test_args.insert(0, "-m")
     else:
         msg = (
             f"Test command {test_command!r} is not supported on Android. "
-            f"Supported commands are 'python -m' and 'python -c'."
+            f"Command must begin with 'python' or 'python3', and contain '-m' or '-c'."
         )
         raise errors.FatalError(msg)
 
@@ -646,6 +649,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None:
         "--cwd",
         cwd_dir,
         *(["-v"] if state.options.build_verbosity > 0 else []),
+        "--",
         *test_args,
         env=state.build_env,
     )

cibuildwheel/resources/_cross_venv.py

@@ -68,16 +68,20 @@ def cross_getandroidapilevel() -> int:
 
     # sysconfig ###############################################################
     #
-    # We don't change the actual sys.base_prefix and base_exec_prefix, because that
-    # could have unpredictable effects. Instead, we change the internal variables
-    # used to generate sysconfig.get_path("include").
-    exec_prefix = sysconfig.get_config_var("exec_prefix")
-    sysconfig._BASE_PREFIX = sysconfig._BASE_EXEC_PREFIX = exec_prefix  # type: ignore[attr-defined]
-
-    # Reload the sysconfigdata file, generating its name from sys.abiflags,
+    # Load the sysconfigdata file, generating its name from sys.abiflags,
     # sys.platform, and sys.implementation._multiarch.
     sysconfig._init_config_vars()  # type: ignore[attr-defined]
 
+    # We don't change the actual sys.base_prefix and base_exec_prefix, because that
+    # could have unpredictable effects. Instead, we change the sysconfig variables
+    # used by sysconfig.get_paths().
+    vars = sysconfig.get_config_vars()
+    try:
+        host_prefix = vars["host_prefix"]  # This variable was added in Python 3.14.
+    except KeyError:
+        host_prefix = vars["exec_prefix"]
+    vars["installed_base"] = vars["installed_platbase"] = host_prefix
+
     # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of
     # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL") (see localized_vars in
     # android.py), and os.uname.

cibuildwheel/resources/build-platforms.toml

@@ -231,8 +231,8 @@ python_configurations = [
 
 [android]
 python_configurations = [
-  { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.7/python-3.13.7-aarch64-linux-android.tar.gz" },
-  { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.7/python-3.13.7-x86_64-linux-android.tar.gz" },
+  { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.8/python-3.13.8-aarch64-linux-android.tar.gz" },
+  { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.8/python-3.13.8-x86_64-linux-android.tar.gz" },
   { identifier = "cp314-android_arm64_v8a", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0-aarch64-linux-android.tar.gz" },
   { identifier = "cp314-android_x86_64", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0-x86_64-linux-android.tar.gz" },
 ]

docs/options.md

@@ -1318,7 +1318,7 @@ The available Pyodide versions are determined by the version of `pyodide-build`
 ### `test-command` {: #test-command env-var toml}
 > The command to test each built wheel
 
-Shell command to run tests after the build. The wheel will be installed
+Command to run tests after the build. The wheel will be installed
 automatically and available for import from the tests. If this variable is not
 set, your wheel will not be installed after building.
 
@@ -1346,11 +1346,12 @@ tree. To access your test code, you have a couple of options:
 
 On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`.
 
-On Android and iOS, the command is parsed by `shlex.split`, and is required to
-be in one of the following forms:
+On Android and iOS, the command is parsed by `shlex.split`, and must be a Python
+command:
 
-* `python -c command ...` (Android only)
-* `python -m module-name ...`
+* On Android, the command must must begin with `python` or `python3`, and contain `-m`
+  or `-c`.
+* On iOS, the command must begin with `python -m`.
 
 Platform-specific environment variables are also available:<br/>
 `CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_ANDROID` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE`

test/_cross_venv_test_android.py

@@ -0,0 +1,20 @@
+import sys
+import sysconfig
+from pathlib import Path
+
+assert sys.platform == "android"
+assert sysconfig.get_platform().startswith("android-")
+
+android_prefix = Path(f"{sys.prefix}/../python/prefix").resolve()
+assert android_prefix.is_dir()
+
+vars = sysconfig.get_config_vars()
+assert vars["INCLUDEDIR"] == f"{android_prefix}/include"
+assert vars["LDVERSION"] == f"{sys.version_info[0]}.{sys.version_info[1]}{sys.abiflags}"
+assert vars["INCLUDEPY"] == f"{vars['INCLUDEDIR']}/python{vars['LDVERSION']}"
+assert vars["LIBDIR"] == f"{android_prefix}/lib"
+assert vars["Py_ENABLE_SHARED"] == 1
+
+paths = sysconfig.get_paths()
+assert paths["include"] == vars["INCLUDEPY"]
+assert paths["platinclude"] == vars["INCLUDEPY"]

test/test_android.py

@@ -32,30 +32,33 @@
         allow_module_level=True,
     )
 
-# Detect CI services which have the Android SDK pre-installed.
-ci_supports_build = (
-    ("CIRRUS_CI" in os.environ and platform.system() == "Darwin")
-    or "GITHUB_ACTIONS" in os.environ
-    or "TF_BUILD" in os.environ  # Azure Pipelines
-)
+# Azure Pipelines does not set the CI variable.
+ci = any(key in os.environ for key in ["CI", "TF_BUILD"])
 
 if "ANDROID_HOME" not in os.environ:
     msg = "ANDROID_HOME environment variable is not set"
-    if ci_supports_build:
+
+    # Fail if we're on a CI service which is supposed to have the Android SDK
+    # pre-installed; otherwise skip the module.
+    if (
+        ("CIRRUS_CI" in os.environ and platform.system() == "Darwin")
+        or "GITHUB_ACTIONS" in os.environ
+        or "TF_BUILD" in os.environ
+    ):
         pytest.fail(msg)
     else:
         pytest.skip(msg, allow_module_level=True)
 
 # Many CI services don't support running the Android emulator: see platforms.md.
-ci_supports_emulator = "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux"
+supports_emulator = (not ci) or ("GITHUB_ACTIONS" in os.environ and platform.system() == "Linux")
 
 
 def needs_emulator(test):
     # All copies of the testbed app run on the same emulator with the same
     # application ID, so these tests must be run serially.
     test = pytest.mark.serial(test)
 
-    if ci_supports_build and not ci_supports_emulator:
+    if not supports_emulator:
         test = pytest.mark.skip("This CI platform doesn't support the emulator")(test)
     return test
 
@@ -92,12 +95,24 @@ def test_android_home(tmp_path, capfd):
     assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err
 
 
-# the first build can fail to setup - mark as flaky, and serial to make sure it runs first
+# android-env.sh may need to install the NDK, and it isn't safe to do that multiple
+# times in parallel. So make sure there's at least one test which gets as far as doing
+# a build, which is marked as serial so it will run before the parallel tests, but isn't
+# marked as needs_emulator so it will run on all CI platforms.
 @pytest.mark.serial
-@pytest.mark.flaky(reruns=2)
-def test_expected_wheels(tmp_path):
-    new_c_project().generate(tmp_path)
-    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
+def test_expected_wheels(tmp_path, spam_env):
+    # Since this test covers all Python versions, check the cross venv.
+    test_module = "_cross_venv_test_android"
+    project = new_c_project(setup_py_add=f"import {test_module}")
+    project.files[f"{test_module}.py"] = (Path(__file__).parent / f"{test_module}.py").read_text()
+    project.generate(tmp_path)
+
+    # Build wheels for all Python versions on the current architecture.
+    del spam_env["CIBW_BUILD"]
+    if not supports_emulator:
+        del spam_env["CIBW_TEST_COMMAND"]
+
+    wheels = cibuildwheel_run(tmp_path, add_env=spam_env)
     assert wheels == expected_wheels(
         "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
     )
@@ -222,20 +237,29 @@ def test_spam():
             print("Spam test passed")
         """
     )
+    project.files["test_empty.py"] = dedent(
+        """\
+        def test_empty():
+            pass
+        """
+    )
+
     project.generate(tmp_path)
 
     return {
         **cp313_env,
-        "CIBW_TEST_SOURCES": "test_spam.py",
+        "CIBW_TEST_SOURCES": "test_spam.py test_empty.py",
         "CIBW_TEST_REQUIRES": "pytest==8.3.5",
+        "CIBW_TEST_COMMAND": "python -m pytest",
     }
 
 
 @needs_emulator
 @pytest.mark.parametrize(
     ("command", "expected_output"),
     [
-        ("python -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"),
+        ("python3 -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"),
+        ("python -m pytest", "=== 2 passed in "),
         ("python -m pytest test_spam.py", "=== 1 passed in "),
         ("pytest test_spam.py", "=== 1 passed in "),
     ],
@@ -252,27 +276,25 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd):
         ) in stderr
 
 
+BAD_FORMAT_ERROR = (
+    "Test command '{}' is not supported on Android. "
+    "Command must begin with 'python' or 'python3', and contain '-m' or '-c'."
+)
+BAD_PLACEHOLDER_ERROR = (
+    "Test command '{}' with a '{{project}}' or '{{package}}' placeholder "
+    "is not supported on Android"
+)
+
+
 @needs_emulator
 @pytest.mark.parametrize(
     ("command", "expected_output"),
     [
-        # Build-time failure: unrecognized command
-        (
-            "./test_spam.py",
-            "Test command './test_spam.py' is not supported on Android. "
-            "Supported commands are 'python -m' and 'python -c'.",
-        ),
-        # Build-time failure: unrecognized placeholder
-        (
-            "pytest {project}",
-            "Test command 'pytest {project}' with a '{project}' or '{package}' "
-            "placeholder is not supported on Android",
-        ),
-        (
-            "pytest {package}",
-            "Test command 'pytest {package}' with a '{project}' or '{package}' "
-            "placeholder is not supported on Android",
-        ),
+        # Build-time failure
+        ("./test_spam.py", BAD_FORMAT_ERROR.format("./test_spam.py")),
+        ("python test_spam.py", BAD_FORMAT_ERROR.format("python test_spam.py")),
+        ("pytest {project}", BAD_PLACEHOLDER_ERROR.format("pytest {project}")),
+        ("pytest {package}", BAD_PLACEHOLDER_ERROR.format("pytest {package}")),
         # Runtime failure
         ("pytest test_ham.py", "not found: test_ham.py"),
     ],
@@ -283,6 +305,29 @@ def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd):
     assert expected_output in capfd.readouterr().err
 
 
+@needs_emulator
+@pytest.mark.parametrize(
+    ("options", "expected"),
+    [
+        ("", 0),
+        ("-E", 1),
+    ],
+)
+def test_test_command_python_options(options, expected, tmp_path, capfd):
+    project = new_c_project()
+    project.generate(tmp_path)
+
+    command = 'import sys; print(f"{sys.flags.ignore_environment=}")'
+    cibuildwheel_run(
+        tmp_path,
+        add_env={
+            **cp313_env,
+            "CIBW_TEST_COMMAND": f"python {options} -c '{command}'",
+        },
+    )
+    assert f"sys.flags.ignore_environment={expected}" in capfd.readouterr().out
+
+
 @needs_emulator
 def test_package_subdir(tmp_path, spam_env, capfd):
     spam_paths = list(tmp_path.iterdir())
@@ -291,17 +336,11 @@ def test_package_subdir(tmp_path, spam_env, capfd):
     for path in spam_paths:
         path.rename(package_dir / path.name)
 
-    test_filename = "package/" + spam_env["CIBW_TEST_SOURCES"]
-    cibuildwheel_run(
-        tmp_path,
-        package_dir,
-        add_env={
-            **spam_env,
-            "CIBW_TEST_SOURCES": test_filename,
-            "CIBW_TEST_COMMAND": f"python -m pytest {test_filename}",
-        },
+    spam_env["CIBW_TEST_SOURCES"] = " ".join(
+        f"package/{path}" for path in spam_env["CIBW_TEST_SOURCES"].split()
     )
-    assert "=== 1 passed in " in capfd.readouterr().out
+    cibuildwheel_run(tmp_path, package_dir, add_env=spam_env)
+    assert "=== 2 passed in " in capfd.readouterr().out
 
 
 @needs_emulator