Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for cross compiling PyPy wheels #687

Merged
merged 2 commits into from
Nov 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 9 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ jobs:
strategy:
matrix:
platform:
# CPython
- target: aarch64-unknown-linux-gnu
arch: aarch64
abi: cp36-cp36m
Expand All @@ -220,14 +221,21 @@ jobs:
- target: s390x-unknown-linux-gnu
arch: s390x
abi: cp310-cp310
# PyPy
- target: aarch64-unknown-linux-gnu
arch: aarch64
abi: pp37-pypy37_pp73
- target: aarch64-unknown-linux-gnu
arch: aarch64
abi: pp38-pypy38_pp73
steps:
- uses: actions/checkout@v2
- name: Build Wheels
run: |
echo 'curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
source ~/.cargo/env
rustup target add ${{ matrix.platform.target }}
export PYO3_CROSS_LIB_DIR=/opt/python/${{ matrix.platform.abi }}/lib
export PYO3_CROSS_LIB_DIR=/opt/python/${{ matrix.platform.abi }}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davidhewitt For #681 I'd like to change pyo3 to support PYO3_CROSS_LIB_DIR without the lib part, what do you think?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @aganders3 in case you are interested in for PyO3/pyo3#1996

cargo run --target x86_64-unknown-linux-gnu -- build -i python3.9 --release --out dist --no-sdist --target ${{ matrix.platform.target }} -m test-crates/pyo3-mixed/Cargo.toml
' > build-wheel.sh
docker run --rm -v "$PWD":/io -w /io messense/manylinux2014-cross:${{ matrix.platform.arch }} bash build-wheel.sh
5 changes: 5 additions & 0 deletions src/build_options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,11 @@ pub fn find_interpreter(
env::set_var("PYTHON_SYS_EXECUTABLE", &host_python.executable);

let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref(), target)?;
env::set_var(
"MATURIN_PYTHON_SYSCONFIGDATA_DIR",
sysconfig_path.parent().unwrap(),
);

let sysconfig_data = parse_sysconfigdata(host_python, sysconfig_path)?;
let major = sysconfig_data
.get("version_major")
Expand Down
4 changes: 4 additions & 0 deletions src/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ fn compile_target(
}
}

if let Some(lib_dir) = env::var_os("MATURIN_PYTHON_SYSCONFIGDATA_DIR") {
build_command.env("PYO3_CROSS_LIB_DIR", lib_dir);
}

let mut cargo_build = build_command.spawn().context("Failed to run cargo")?;

let mut artifacts = HashMap::new();
Expand Down
13 changes: 9 additions & 4 deletions src/cross_compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,12 @@ pub fn find_sysconfigdata(lib_dir: &Path, target: &Target) -> Result<PathBuf> {
/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
fn search_lib_dir(path: impl AsRef<Path>, target: &Target) -> Vec<PathBuf> {
let mut sysconfig_paths = vec![];
let version_pat = if let Some(v) =
let (cpython_version_pat, pypy_version_pat) = if let Some(v) =
env::var_os("PYO3_CROSS_PYTHON_VERSION").map(|s| s.into_string().unwrap())
{
format!("python{}", v)
(format!("python{}", v), format!("pypy{}", v))
} else {
"python3.".into()
("python3.".into(), "pypy3.".into())
};
for f in fs::read_dir(path.as_ref()).expect("Path does not exist") {
let sysc = match &f {
Expand All @@ -173,7 +173,12 @@ fn search_lib_dir(path: impl AsRef<Path>, target: &Target) -> Vec<PathBuf> {
}
search_lib_dir(f.path(), target)
}
Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), target),
Ok(f) if starts_with(f, &cpython_version_pat) => search_lib_dir(f.path(), target),
// PyPy 3.7: /opt/python/pp37-pypy37_pp73/lib_pypy/_sysconfigdata__linux_x86_64-linux-gnu.py
Ok(f) if starts_with(f, "lib_pypy") => search_lib_dir(f.path(), target),
// PyPy 3.8: /opt/python/pp38-pypy38_pp73/lib/pypy3.8/_sysconfigdata__linux_x86_64-linux-gnu.py
Ok(f) if starts_with(f, &pypy_version_pat) => search_lib_dir(f.path(), target),
Ok(f) if starts_with(f, "lib") && f.path().is_dir() => search_lib_dir(f.path(), target),
_ => continue,
};
sysconfig_paths.extend(sysc);
Expand Down