Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,8 @@ Some of the functionality of `pyo3-build-config`:
entirely and only abi3 extensions can be built.
- Check if we are building a Python extension.
- If we are building an extension (e.g., Python library installable by `pip`),
we don't link `libpython`.
Currently we use the `extension-module` feature for this purpose. This may change in the future.
See [#1123](https://github.com/PyO3/pyo3/pull/1123).
we don't link `libpython` on most platforms (to allow for statically-linked Python interpreters).
The `PYO3_BUILD_EXTENSION_MODULE` environment variable suppresses linking.
- Cross-compiling configuration
- If `TARGET` architecture and `HOST` architecture differ, we can find cross compile information
from environment variables (`PYO3_CROSS_LIB_DIR`, `PYO3_CROSS_PYTHON_VERSION` and
Expand Down
6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,9 @@ macros = ["pyo3-macros", "indoc", "unindent"]
# Enables multiple #[pymethods] per #[pyclass]
multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"]

# Use this feature when building an extension module.
# It tells the linker to keep the python symbols unresolved,
# so that the module can also be used with statically linked python interpreters.
# Deprecated: use the `PYO3_BUILD_EXTENSION_MODULE` environment variable when
# building a Python extension module (set automatically by `setuptools-rust` and
# `maturin`).
extension-module = ["pyo3-ffi/extension-module"]

# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ name = "string_sum"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "0.27.1", features = ["extension-module"] }
pyo3 = "0.27.1"
```

**`src/lib.rs`**
Expand Down Expand Up @@ -138,6 +138,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th
```toml
[dependencies.pyo3]
version = "0.27.1"
# Enabling this cargo feature will cause PyO3 to start a Python interpreter on first call to `Python::attach`
features = ["auto-initialize"]
```

Expand Down
2 changes: 1 addition & 1 deletion examples/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ edition = "2021"
rust-version = "1.83"

[dev-dependencies]
pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] }
pyo3 = { path = "..", features = ["auto-initialize"] }

[[example]]
name = "decorator"
Expand Down
2 changes: 1 addition & 1 deletion examples/decorator/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ name = "decorator"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3 = "{{PYO3_VERSION}}"
2 changes: 1 addition & 1 deletion examples/decorator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ name = "decorator"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { path = "../../", features = ["extension-module"] }
pyo3 = { path = "../../" }

[workspace]
2 changes: 1 addition & 1 deletion examples/getitem/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ name = "getitem"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3 = "{{PYO3_VERSION}}"
2 changes: 1 addition & 1 deletion examples/getitem/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ name = "getitem"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { path = "../../", features = ["extension-module"] }
pyo3 = { path = "../../" }

[workspace]
2 changes: 1 addition & 1 deletion examples/maturin-starter/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ name = "maturin_starter"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3 = "{{PYO3_VERSION}}"
2 changes: 1 addition & 1 deletion examples/maturin-starter/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ name = "maturin_starter"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { path = "../../", features = ["extension-module"] }
pyo3 = { path = "../../" }

[features]
abi3 = ["pyo3/abi3-py37", "generate-import-lib"]
Expand Down
5 changes: 0 additions & 5 deletions examples/plugin/.template/plugin_api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,4 @@ name = "plugin_api"
crate-type = ["cdylib", "rlib"]

[dependencies]
#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
pyo3 = "{{PYO3_VERSION}}"

[features]
# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
extension-module = ["pyo3/extension-module"]
8 changes: 4 additions & 4 deletions examples/plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ interface types that can be used to exchange data between Rust and Python. This

# Building and Testing
## Host application
To run the app itself, you only need to run
To run the app itself, you only need to run

```shell
cargo run
Expand All @@ -14,20 +14,20 @@ It will build the app, as well as the plugin API, then run the app, load the plu

## Plugin API testing

The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app.
The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app.

To build the API only package, first install `maturin`:

```shell
pip install maturin
```

When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3.
When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3.
Instead, one needs to enable the optional feature as follows:

```shell
cd plugin_api
maturin build --features "extension-module"
maturin build"
```

Alternatively, install nox and run the tests inside an isolated environment:
Expand Down
5 changes: 0 additions & 5 deletions examples/plugin/plugin_api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,4 @@ name = "plugin_api"
crate-type = ["cdylib", "rlib"]

[dependencies]
#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
pyo3 = { path = "../../../" }

[features]
# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
extension-module = ["pyo3/extension-module"]
2 changes: 1 addition & 1 deletion examples/setuptools-rust-starter/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ name = "setuptools_rust_starter"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3 = "{{PYO3_VERSION}}"
2 changes: 1 addition & 1 deletion examples/setuptools-rust-starter/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ name = "setuptools_rust_starter"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { path = "../../", features = ["extension-module"] }
pyo3 = { path = "../../" }

[workspace]
2 changes: 1 addition & 1 deletion examples/word-count/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ name = "word_count"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3 = "{{PYO3_VERSION}}"
rayon = "1.0.2"
2 changes: 1 addition & 1 deletion examples/word-count/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ name = "word_count"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { path = "../..", features = ["extension-module"] }
pyo3 = { path = "../.." }
rayon = "1.0.2"

[workspace]
30 changes: 18 additions & 12 deletions guide/src/building-and-distribution.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,13 @@ Package vendors can opt-in to the "abi3" limited Python API which allows their w
There are many ways to go about this: it is possible to use `cargo` to build the extension module (along with some manual work, which varies with OS).
The PyO3 ecosystem has two packaging tools, [`maturin`] and [`setuptools-rust`], which abstract over the OS difference and also support building wheels for PyPI upload.

PyO3 has some Cargo features to configure projects for building Python extension modules:
PyO3 has some functionality for configuring projects when building Python extension modules:

- The `extension-module` feature, which must be enabled when building Python extension modules.
- The `abi3` feature and its version-specific `abi3-pyXY` companions, which are used to opt-in to the limited Python API in order to support multiple Python versions in a single wheel.
- The `PYO3_BUILD_EXTENSION_MODULE` environment variable, which must be set when building Python extension modules. `maturin` and `setuptools-rust` set this automatically.
- The `abi3` Cargo feature and its version-specific `abi3-pyXY` companions, which are used to opt-in to the limited Python API in order to support multiple Python versions in a single wheel.

This section describes each of these packaging tools before describing how to build manually without them.
It then proceeds with an explanation of the `extension-module` feature.
This section describes the packaging tools before describing how to build manually without them.
It then proceeds with an explanation of the `PYO3_BUILD_EXTENSION_MODULE` environment variable.
Finally, there is a section describing PyO3's `abi3` features.

### Packaging tools
Expand All @@ -97,7 +97,7 @@ There are also [`maturin-starter`] and [`setuptools-rust-starter`] examples in t

### Manual builds

To build a PyO3-based Python extension manually, start by running `cargo build` as normal in a library project which uses PyO3's `extension-module` feature and has the [`cdylib` crate type](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-crate-type-field).
To build a PyO3-based Python extension manually, start by running `cargo build` as normal in a library project with the [`cdylib` crate type](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-crate-type-field) while the `PYO3_BUILD_EXTENSION_MODULE` environment variable is set.

Once built, symlink (or copy) and rename the shared library from Cargo's `target/` directory to your desired output directory:

Expand Down Expand Up @@ -142,7 +142,7 @@ See [PEP 3149](https://peps.python.org/pep-3149/) for more background on platfor

#### macOS

On macOS, because the `extension-module` feature disables linking to `libpython` ([see the next section](#the-extension-module-feature)), some additional linker arguments need to be set. `maturin` and `setuptools-rust` both pass these arguments for PyO3 automatically, but projects using manual builds will need to set these directly in order to support macOS.
On macOS, because the `PYO3_BUILD_EXTENSION_MODULE` environment variable disables linking to `libpython` ([see the next section](#the-extension-module-feature)), some additional linker arguments need to be set. `maturin` and `setuptools-rust` both pass these arguments for PyO3 automatically, but projects using manual builds will need to set these directly in order to support macOS.

The easiest way to set the correct linker arguments is to add a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) with the following content:

Expand Down Expand Up @@ -193,17 +193,23 @@ For more discussion on and workarounds for MacOS linking problems [see this issu

Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)).

### The `extension-module` feature
### The `PYO3_BUILD_EXTENSION_MODULE` environment variable

PyO3's `extension-module` feature is used to disable [linking](https://en.wikipedia.org/wiki/Linker_(computing)) to `libpython` on Unix targets.
<a name="the-extension-module-feature"></a> <!-- for backwards compatibility -->

This is necessary because by default PyO3 links to `libpython`.
By default PyO3 links to `libpython`.
This makes binaries, tests, and examples "just work".
However, Python extensions on Unix must not link to libpython for [manylinux](https://www.python.org/dev/peps/pep-0513/) compliance.

The downside of not linking to `libpython` is that binaries, tests, and examples (which usually embed Python) will fail to build.
If you have an extension module as well as other outputs in a single project, you need to use optional Cargo features to disable the `extension-module` when you're not building the extension module.
See [the FAQ](faq.md#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror) for an example workaround.
As a result, PyO3 uses an envionment variable `PYO3_BUILD_EXTENSION_MODULE` to disable linking to `libpython`.
This should only be set when building a library for distribution.
`maturin >= 1.9.4` and `setuptools-rust >= 1.12` will set this for you automatically.

> Note: historically PyO3 used an `extension-module` feature to perform the same function now done by the `PYO3_BUILD_EXTENSION_MODULE` env var.
> This feature caused linking to be disabled for all compile targets, including Rust tests and benchmarks.
>
> Projects are encouraged to migrate off the feature, as it caused [major development pain](faq.md#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror) due to the lack of linking.

### `Py_LIMITED_API`/`abi3`

Expand Down
39 changes: 5 additions & 34 deletions guide/src/faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,47 +26,18 @@ See the documentation for [`PyOnceLock`] and [`OnceExt`] for further details and

## I can't run `cargo test`; or I can't build in a Cargo workspace: I'm having linker issues like "Symbol not found" or "Undefined reference to _PyExc_SystemError"

Currently, [#340](https://github.com/PyO3/pyo3/issues/340) causes `cargo test` to fail with linking errors when the `extension-module` feature is activated.
Linking errors can also happen when building in a cargo workspace where a different crate also uses PyO3 (see [#2521](https://github.com/PyO3/pyo3/issues/2521)).
For now, there are three ways we can work around these issues.
The `extension-module` feature (now deprecated) disables linking to `libpython`.
This breaks binaries and tests which need to load symbols from `libpython` to execute.

1. Make the `extension-module` feature optional.
Build with `maturin develop --features "extension-module"`
Remove the `extension-module` feature and upgrade to `maturin >= 1.9.4` or `setuptools-rust 1.12`.

```toml
[dependencies.pyo3]
{{#PYO3_CRATE_VERSION}}

[features]
extension-module = ["pyo3/extension-module"]
```

2. Make the `extension-module` feature optional and default.
Run tests with `cargo test --no-default-features`:

```toml
[dependencies.pyo3]
{{#PYO3_CRATE_VERSION}}

[features]
extension-module = ["pyo3/extension-module"]
default = ["extension-module"]
```

3. If you are using a [`pyproject.toml`](https://maturin.rs/metadata.html) file to control maturin settings, add the following section:

```toml
[tool.maturin]
features = ["pyo3/extension-module"]
# Or for maturin 0.12:
# cargo-extra-args = ["--features", "pyo3/extension-module"]
```
If building manually, see the [`PYO3_BUILD_EXTENSION_MODULE` environment variable](./building-and-distribution.md#the-pyo3_build_extension_module-environment-variable).

## I can't run `cargo test`: my crate cannot be found for tests in `tests/` directory

The Rust book suggests to [put integration tests inside a `tests/` directory](https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests).

For a PyO3 `extension-module` project where the `crate-type` is set to `"cdylib"` in your `Cargo.toml`, the compiler won't be able to find your crate and will display errors such as `E0432` or `E0463`:
For a PyO3 project where the `crate-type` is set to `"cdylib"` in your `Cargo.toml`, the compiler won't be able to find your crate and will display errors such as `E0432` or `E0463`:

```text
error[E0432]: unresolved import `my_crate`
Expand Down
4 changes: 1 addition & 3 deletions guide/src/features.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ By default, only the `macros` feature is enabled.

### `extension-module`

This feature is required when building a Python extension module using PyO3.

It tells PyO3's build script to skip linking against `libpython.so` on Unix platforms, where this must not be done.
Deprecated, users should remove this feature and upgrade to `maturin >= 1.9.4` or `setuptools-rust >= 1.12`.

See the [building and distribution](building-and-distribution.md#the-extension-module-feature) section for further detail.

Expand Down
4 changes: 2 additions & 2 deletions guide/src/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ name = "pyo3_example"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["extension-module"] }
pyo3 = {{#PYO3_CRATE_VERSION}}
```

## pyproject.toml
Expand All @@ -147,7 +147,7 @@ You should also create a `pyproject.toml` with the following contents:

```toml
[build-system]
requires = ["maturin>=1,<2"]
requires = ["maturin>=1.9.4,<2"]
build-backend = "maturin"

[project]
Expand Down
2 changes: 1 addition & 1 deletion pyo3-build-config/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ default = []
# script. If this feature isn't enabled, the build script no-ops.
resolve-config = []

# This feature is enabled by pyo3 when building an extension module.
# deprecated
extension-module = []

# Automatically generates `python3.dll` import libraries for Windows targets.
Expand Down
2 changes: 1 addition & 1 deletion pyo3-build-config/src/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -872,7 +872,7 @@ pub fn get_abi3_version() -> Option<PythonVersion> {
/// Checks if the `extension-module` feature is enabled for the PyO3 crate.
///
/// This can be triggered either by:
/// - The `extension-module` Cargo feature
/// - The `extension-module` Cargo feature (deprecated)
/// - Setting the `PYO3_BUILD_EXTENSION_MODULE` environment variable
///
/// Must be called from a PyO3 crate build script.
Expand Down
2 changes: 1 addition & 1 deletion pyo3-build-config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub fn use_pyo3_cfgs() {
}
}

/// Adds linker arguments suitable for PyO3's `extension-module` feature.
/// Adds linker arguments suitable for linking an extension module.
///
/// This should be called from a build script.
///
Expand Down
5 changes: 1 addition & 4 deletions pyo3-ffi-check/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,9 @@ publish = false

[dependencies]
pyo3-ffi-check-macro = { path = "./macro" }
pyo3-ffi = { path = "../pyo3-ffi" }
memoffset = "0.9.0"

[dependencies.pyo3-ffi]
path = "../pyo3-ffi"
features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols)

[build-dependencies]
bindgen = "0.69.4"
pyo3-build-config = { path = "../pyo3-build-config" }
Expand Down
4 changes: 1 addition & 3 deletions pyo3-ffi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ libc = "0.2.62"

default = []

# Use this feature when building an extension module.
# It tells the linker to keep the python symbols unresolved,
# so that the module can also be used with statically linked python interpreters.
# deprecated
extension-module = ["pyo3-build-config/extension-module"]

# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
Expand Down
5 changes: 2 additions & 3 deletions pyo3-ffi/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,8 @@ name = "string_sum"
# crate-type = ["cdylib", "rlib"]
crate-type = ["cdylib"]

[dependencies.pyo3-ffi]
version = "0.27.1"
features = ["extension-module"]
[dependencies]
pyo3-ffi = "0.27.1"

[build-dependencies]
# This is only necessary if you need to configure your build based on
Expand Down
2 changes: 1 addition & 1 deletion pyo3-ffi/examples/sequential/.template/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ name = "sequential"
crate-type = ["cdylib", "lib"]

[dependencies]
pyo3-ffi = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
pyo3-ffi = { version = "{{PYO3_VERSION}}" }
Loading
Loading