From 4cbdee2701519516cf7689a32fabc7a09198dab7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 3 Dec 2025 15:16:59 -0500 Subject: [PATCH 1/3] docs: mention names of modules Signed-off-by: Henry Schreiner --- docs/_includes/pyproject.md | 3 ++- docs/pages/guides/packaging_compiled.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/_includes/pyproject.md b/docs/_includes/pyproject.md index a747061b..d8ae216d 100644 --- a/docs/_includes/pyproject.md +++ b/docs/_includes/pyproject.md @@ -53,7 +53,8 @@ Changelog = "https://github.com/org/package/releases" -You can read more about each field, and all allowed fields, in +In this example, `"package"` is the name of the thing you are working on. You +can read more about each field, and all allowed fields, in [packaging.python.org][metadata], [Flit](https://flit.readthedocs.io/en/latest/pyproject_toml.html#new-style-metadata) or [Whey](https://whey.readthedocs.io/en/latest/configuration.html). Only the diff --git a/docs/pages/guides/packaging_compiled.md b/docs/pages/guides/packaging_compiled.md index 337c4053..fc5d6005 100644 --- a/docs/pages/guides/packaging_compiled.md +++ b/docs/pages/guides/packaging_compiled.md @@ -203,6 +203,11 @@ features = ["extension-module", "abi3-py310"] ## Example compiled file +This example will make a `_core` extension inside your package; this pattern +allows you to easily provide both Python files and compiled extensions, and +keeping the details of your compiled extension private. You can select whatever +name you wish, though, or even make your compiled extension a top level module. + {% tabs %} {% tab skbc Scikit-build-core %} Example `src/main.cpp` file: From 437f162030e636c5c6880ae2934b938677bea013 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 3 Dec 2025 16:41:24 -0500 Subject: [PATCH 2/3] fix: modernize Rust example, and render into docs Signed-off-by: Henry Schreiner --- docs/pages/guides/packaging_compiled.md | 151 +++++++++++++----- helpers/cog_helpers.py | 8 + ...ter.backend=='maturin' %}lib.rs{% endif %} | 33 ++-- ...backend=='maturin' %}Cargo.toml{% endif %} | 6 +- 4 files changed, 141 insertions(+), 57 deletions(-) diff --git a/docs/pages/guides/packaging_compiled.md b/docs/pages/guides/packaging_compiled.md index fc5d6005..c5b5afcb 100644 --- a/docs/pages/guides/packaging_compiled.md +++ b/docs/pages/guides/packaging_compiled.md @@ -8,6 +8,24 @@ parent: Topical Guides {% include toc.html %} + + + # Packaging Compiled Projects There are a variety of ways to package compiled projects. In the past, the only @@ -52,27 +70,48 @@ selects the backend: {% tabs %} {% tab skbc Scikit-build-core %} + + ```toml [build-system] -requires = ["scikit-build-core"] +requires = ["pybind11", "scikit-build-core>=0.11"] build-backend = "scikit_build_core.build" ``` + + {% endtab %} {% tab meson Meson-python %} + + ```toml [build-system] -requires = ["meson-python"] +requires = ["meson-python>=0.18", "pybind11"] build-backend = "mesonpy" ``` + + {% endtab %} {% tab maturin Maturin %} + + ```toml [build-system] -requires = ["maturin"] +requires = ["maturin>=1.9,<2"] build-backend = "maturin" ``` + + {% endtab %} {% endtabs %} @@ -83,20 +122,49 @@ build-backend = "maturin" These tools all read the project table. They also have extra configuration options in `tool.*` settings. +{% tabs %} {% tab skbc Scikit-build-core %} + + +```toml +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" +``` + +These options are not required, but can improve your experience. + +{% endtab %} {% tab meson Meson-python %} + +No `tool.meson-python` configuration required for this example. + +{% endtab %} {% tab maturin Maturin %} + + + +```toml +[tool.maturin] +module-name = "package._core" +python-packages = ["package"] +python-source = "src" +sdist-generator = "git" # default is cargo +``` + + + +Maturin assumes you follow Rust's package structure, so we need a little bit of +configuration here to follow the convention of the other tools here. + +{% endtab %} {% endtabs %} + ## Backend specific files {% tabs %} {% tab skbc Scikit-build-core %} @@ -180,7 +248,7 @@ with code_fence("toml"): [package] name = "package" version = "0.1.0" -edition = "2018" +edition = "2021" [lib] name = "_core" @@ -188,10 +256,10 @@ name = "_core" crate-type = ["cdylib"] [dependencies] -rand = "0.8.3" +rand = "0.9.2" [dependencies.pyo3] -version = "0.19.1" +version = "0.27.2" # "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so) # "abi3-py310" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.10 features = ["extension-module", "abi3-py310"] @@ -302,26 +370,29 @@ with code_fence("rs"): ```rs use pyo3::prelude::*; -#[pyfunction] -fn add(x: i64, y: i64) -> i64 { - x + y -} - -#[pyfunction] -fn subtract(x: i64, y: i64) -> i64 { - x - y -} - /// A Python module implemented in Rust. The name of this function must match /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. #[pymodule] -fn _core(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(add, m)?)?; - m.add_function(wrap_pyfunction!(subtract, m)?)?; - m.add("__version__", env!("CARGO_PKG_VERSION"))?; +mod _core { + use super::*; + + #[pyfunction] + fn add(x: i64, y: i64) -> i64 { + x + y + } + + #[pyfunction] + fn subtract(x: i64, y: i64) -> i64 { + x - y + } + - Ok(()) + #[pymodule_init] + fn pymodule_init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + Ok(()) + } } ``` @@ -332,7 +403,8 @@ fn _core(_py: Python, m: &PyModule) -> PyResult<()> { ## Package structure The recommendation (followed above) is to have source code in `/src`, and the -Python package files in `/src/`. +Python package files in `/src/`. The compiled files also can go in +`/src`. ## Versioning @@ -348,25 +420,26 @@ though the defaults are reasonable. Unlike pure Python, you'll need to build redistributable wheels for each platform and supported Python version if you want to avoid compilation on the -user's system. See [the CI page on wheels][gha_wheels] for a suggested workflow. +user's system using cibuildwheel. See [the CI page on wheels][gha_wheels] for a +suggested workflow. ## Special considerations ### NumPy Modern versions of NumPy (1.25+) allow you to target older versions when -building, which is _highly_ recommended, and this will become required in NumPy -2.0. Now you add: +building, which is _highly_ recommended, and this became required in NumPy 2.0. +Now you add: ```cpp #define NPY_TARGET_VERSION NPY_1_22_API_VERSION ``` (Where that number is whatever version you support as a minimum) then make sure -you build with NumPy 1.25+ (or 2.0+ when it comes out). Before 1.25, it was -necessary to actually pin the oldest NumPy you supported (the -`oldest-supported-numpy` package is the easiest method). If you support Python < -3.9, you'll have to use the old method for those versions. +you build with NumPy 1.25+ (or 2.0+). Before 1.25, it was necessary to actually +pin the oldest NumPy you supported (the `oldest-supported-numpy` package is the +easiest method). If you support Python < 3.9, you'll have to use the old method +for those versions. If using pybind11, you don't need NumPy at build-time in the first place. diff --git a/helpers/cog_helpers.py b/helpers/cog_helpers.py index 3e935b32..765229f1 100644 --- a/helpers/cog_helpers.py +++ b/helpers/cog_helpers.py @@ -73,6 +73,14 @@ def get_source(self, dotted_name: str, /) -> str: ) return tomlkit.dumps(toml).strip() + def __contains__(self, dotted_name: str, /) -> bool: + names = dotted_name.split(".") + try: + functools.reduce(lambda d, k: d[k], names, self.toml) + except KeyError: + return False + return True + @contextlib.contextmanager def code_fence(lang: str, /, *, width: int = 3) -> Generator[None, None, None]: diff --git a/{{cookiecutter.project_name}}/src/{% if cookiecutter.backend=='maturin' %}lib.rs{% endif %} b/{{cookiecutter.project_name}}/src/{% if cookiecutter.backend=='maturin' %}lib.rs{% endif %} index 314fedc8..8307f233 100644 --- a/{{cookiecutter.project_name}}/src/{% if cookiecutter.backend=='maturin' %}lib.rs{% endif %} +++ b/{{cookiecutter.project_name}}/src/{% if cookiecutter.backend=='maturin' %}lib.rs{% endif %} @@ -1,23 +1,26 @@ use pyo3::prelude::*; -#[pyfunction] -fn add(x: i64, y: i64) -> i64 { - x + y -} - -#[pyfunction] -fn subtract(x: i64, y: i64) -> i64 { - x - y -} - /// A Python module implemented in Rust. The name of this function must match /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. #[pymodule] -fn _core(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(add, m)?)?; - m.add_function(wrap_pyfunction!(subtract, m)?)?; - m.add("__version__", env!("CARGO_PKG_VERSION"))?; +mod _core { + use super::*; + + #[pyfunction] + fn add(x: i64, y: i64) -> i64 { + x + y + } + + #[pyfunction] + fn subtract(x: i64, y: i64) -> i64 { + x - y + } + - Ok(()) + #[pymodule_init] + fn pymodule_init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + Ok(()) + } } diff --git a/{{cookiecutter.project_name}}/{% if cookiecutter.backend=='maturin' %}Cargo.toml{% endif %} b/{{cookiecutter.project_name}}/{% if cookiecutter.backend=='maturin' %}Cargo.toml{% endif %} index b4e6a693..002d305c 100644 --- a/{{cookiecutter.project_name}}/{% if cookiecutter.backend=='maturin' %}Cargo.toml{% endif %} +++ b/{{cookiecutter.project_name}}/{% if cookiecutter.backend=='maturin' %}Cargo.toml{% endif %} @@ -1,7 +1,7 @@ [package] name = "{{ cookiecutter.__project_slug }}" version = "0.1.0" -edition = "2018" +edition = "2021" [lib] name = "_core" @@ -9,10 +9,10 @@ name = "_core" crate-type = ["cdylib"] [dependencies] -rand = "0.8.3" +rand = "0.9.2" [dependencies.pyo3] -version = "0.19.1" +version = "0.27.2" # "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so) # "abi3-py310" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.10 features = ["extension-module", "abi3-py310"] From ff9f8b839e3890411da7ed02b78ddc5b0a27ea45 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 3 Dec 2025 16:58:13 -0500 Subject: [PATCH 3/3] fix: drop one unneeded line Signed-off-by: Henry Schreiner --- docs/pages/guides/packaging_compiled.md | 1 - {{cookiecutter.project_name}}/pyproject.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/pages/guides/packaging_compiled.md b/docs/pages/guides/packaging_compiled.md index c5b5afcb..b39fbc7d 100644 --- a/docs/pages/guides/packaging_compiled.md +++ b/docs/pages/guides/packaging_compiled.md @@ -153,7 +153,6 @@ with code_fence("toml"): ```toml [tool.maturin] module-name = "package._core" -python-packages = ["package"] python-source = "src" sdist-generator = "git" # default is cargo ``` diff --git a/{{cookiecutter.project_name}}/pyproject.toml b/{{cookiecutter.project_name}}/pyproject.toml index acd693b3..5133899a 100644 --- a/{{cookiecutter.project_name}}/pyproject.toml +++ b/{{cookiecutter.project_name}}/pyproject.toml @@ -162,7 +162,6 @@ sdist.include = ["src/{{ cookiecutter.__project_slug }}/_version.py"] [tool.maturin] module-name = "{{ cookiecutter.__project_slug }}._core" -python-packages = ["{{ cookiecutter.__project_slug }}"] python-source = "src" sdist-generator = "git" # default is cargo