From 755bf97fdbe9399cfbd072ff5e9a26c81d5890dc Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Fri, 7 Jan 2022 20:44:36 -0500 Subject: [PATCH 01/11] Add export-config feature to pyo3-build-config. --- pyo3-build-config/Cargo.toml | 3 ++ pyo3-build-config/src/impl_.rs | 73 +++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 2233dd78a4e..88c1b1df384 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -20,6 +20,9 @@ default = [] # script. If this feature isn't enabled, the build script no-ops. resolve-config = [] +# Export the resolved build config into a file for use by e.g. other build scripts. +export-config = ["resolve-config"] + abi3 = [] abi3-py37 = ["abi3-py38"] abi3-py38 = ["abi3-py39"] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index f18221ae6d4..243c0df394d 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -4,7 +4,7 @@ use std::{ env, ffi::{OsStr, OsString}, fmt::Display, - fs::{self, DirEntry}, + fs::{self, DirEntry, File}, io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -21,6 +21,8 @@ use crate::{ const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; /// Maximum Python version that can be used as minimum required Python version with abi3. const ABI3_MAX_MINOR: u8 = 9; +/// Name of config file exported by "export-config" feature. +const EXPORT_CONFIG_FILENAME: &str = ".pyo3-export-config"; /// Gets an environment variable owned by cargo. /// @@ -155,6 +157,24 @@ impl InterpreterConfig { for flag in &self.build_flags.0 { println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag) } + + if cfg!(feature = "export-config") { + let output_path = + Path::new(&cargo_env_var("OUT_DIR").unwrap()).join(EXPORT_CONFIG_FILENAME); + if let Ok(config_file) = File::create(&output_path) { + if self.to_writer(config_file).is_err() { + warn!( + "Failed to export build config - this may be a problem for external crates that \ + depend on the pyo3 config." + ); + } else { + println!( + "cargo:PYO3_EXPORT_CONFIG={}", + output_path.canonicalize().unwrap().to_str().unwrap() + ); + } + } + } } #[doc(hidden)] @@ -339,6 +359,14 @@ print("mingw", get_platform().startswith("mingw")) InterpreterConfig::from_reader(reader) } + /// Create an InterpreterConfig generated from pyo3-build-config using the "export-config" + /// feature + pub fn from_pyo3_export_config() -> Result { + InterpreterConfig::from_path(Path::new( + &cargo_env_var("DEP_PYTHON_PYO3_EXPORT_CONFIG").unwrap(), + )) + } + #[doc(hidden)] pub fn from_reader(reader: impl Read) -> Result { let reader = BufReader::new(reader); @@ -461,6 +489,37 @@ print("mingw", get_platform().startswith("mingw")) } Ok(()) } + + /// Run a python script using the executable of this InterpreterConfig with additional + /// environment variables (e.g. PYTHONPATH) set. + pub fn run_python_script_with_envs(&self, script: &str, envs: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + run_python_script_with_envs( + Path::new( + self.executable + .as_ref() + .expect("No interpreter executable!"), + ), + script, + envs, + ) + } + + /// Run a python script using the executable of this InterpreterConfig. + pub fn run_python_script(&self, script: &str) -> Result { + run_python_script( + Path::new( + self.executable + .as_ref() + .expect("No interpreter executable!"), + ), + script, + ) + } } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -1183,8 +1242,20 @@ fn default_lib_name_unix( /// Run a python script using the specified interpreter binary. fn run_python_script(interpreter: &Path, script: &str) -> Result { + run_python_script_with_envs(interpreter, script, std::iter::empty::<(&str, &str)>()) +} + +/// Run a python script using the specified interpreter binary with additional environment +/// variables (e.g. PYTHONPATH) set. +fn run_python_script_with_envs(interpreter: &Path, script: &str, envs: I) -> Result +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ let out = Command::new(interpreter) .env("PYTHONIOENCODING", "utf-8") + .envs(envs) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) From cb3effdb96ac576038314782cbf40613b68e5db1 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Fri, 7 Jan 2022 21:21:16 -0500 Subject: [PATCH 02/11] Additon conditional compilation for InterpreterConfig::from_pyo3_export_config --- pyo3-build-config/src/impl_.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 243c0df394d..b3c48eec5d4 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -361,6 +361,7 @@ print("mingw", get_platform().startswith("mingw")) /// Create an InterpreterConfig generated from pyo3-build-config using the "export-config" /// feature + #[cfg(feature = "export-config")] pub fn from_pyo3_export_config() -> Result { InterpreterConfig::from_path(Path::new( &cargo_env_var("DEP_PYTHON_PYO3_EXPORT_CONFIG").unwrap(), From 03e11184ed68b8728e41610708baff221152d28e Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 13:30:54 -0500 Subject: [PATCH 03/11] Add changelog entry. Rebased on main. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bded73909b..f8c6941b3de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Allow to allow dependent crates to access values from `pyo3-build-config` via cargo link env vars. [#2092](https://github.com/PyO3/pyo3/pull/2092) + ### Fixed - Considered `PYTHONFRAMEWORK` when cross compiling in order that on macos cross compiling against a [Framework bundle](https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html) is considered shared. [#2233](https://github.com/PyO3/pyo3/pull/2233) From ef4b91256df11e07712e80698c686ee4a431d60a Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 13:31:17 -0500 Subject: [PATCH 04/11] Add tests for run_python_script*. --- pyo3-build-config/src/impl_.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index b3c48eec5d4..bf22a771917 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1920,4 +1920,29 @@ mod tests { .is_none() ); } + + #[test] + fn test_run_python_script() { + // as above, this should be okay in CI where Python is presumed installed + let interpreter = make_interpreter_config() + .expect("could not get InterpreterConfig from installed interpreter"); + let out = interpreter + .run_python_script("print(2 + 2)") + .expect("failed to run Python script"); + assert_eq!(out, "4\n"); + } + + #[test] + fn test_run_python_script_with_envs() { + // as above, this should be okay in CI where Python is presumed installed + let interpreter = make_interpreter_config() + .expect("could not get InterpreterConfig from installed interpreter"); + let out = interpreter + .run_python_script_with_envs( + "import os; print(os.getenv('PYO3_TEST'))", + [("PYO3_TEST", "42")], + ) + .expect("failed to run Python script"); + assert_eq!(out, "42\n"); + } } From 04b8652612f2d83d1cdcef7718ff176982a88b13 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 13:43:11 -0500 Subject: [PATCH 05/11] Fix test on windows (line ending mismatch). --- pyo3-build-config/src/impl_.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index bf22a771917..e8dbf50b349 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1929,7 +1929,7 @@ mod tests { let out = interpreter .run_python_script("print(2 + 2)") .expect("failed to run Python script"); - assert_eq!(out, "4\n"); + assert_eq!(out.trim_end(), "4"); } #[test] @@ -1943,6 +1943,6 @@ mod tests { [("PYO3_TEST", "42")], ) .expect("failed to run Python script"); - assert_eq!(out, "42\n"); + assert_eq!(out.trim_end(), "42"); } } From 14bd6282ad44a52465530a663124d6652a8ee03f Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 15:07:46 -0500 Subject: [PATCH 06/11] Add test for export-config round trip. --- .github/workflows/ci.yml | 2 +- pyo3-build-config/src/impl_.rs | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e0cd74a7c6..589e3ad3012 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,7 +227,7 @@ jobs: run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml - name: Test build config - run: cargo test --manifest-path=pyo3-build-config/Cargo.toml + run: cargo test --manifest-path=pyo3-build-config/Cargo.toml --all-features - name: Test python examples and tests shell: bash diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index e8dbf50b349..dc147467829 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1945,4 +1945,19 @@ mod tests { .expect("failed to run Python script"); assert_eq!(out.trim_end(), "42"); } + + #[test] + #[cfg(feature = "export-config")] + fn test_emit_pyo3_configs_round_trip() { + let interpreter = make_interpreter_config() + .expect("could not get InterpreterConfig from installed interpreter"); + interpreter.emit_pyo3_cfgs(); + // config path env var is not set during testing of this crate alone, as it relies on the + // pyo3 python links manifest key, so we have to hard-code the same path + let output_path = + Path::new(&cargo_env_var("OUT_DIR").unwrap()).join(EXPORT_CONFIG_FILENAME); + let reimported_interpreter = InterpreterConfig::from_path(output_path) + .expect("could not generated InterpreterConfig from exported config"); + assert_eq!(interpreter, reimported_interpreter); + } } From 161d79fae3053028a77f7362ed2e161bf1efa24f Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 15:39:19 -0500 Subject: [PATCH 07/11] Fix test for MSRV --- pyo3-build-config/src/impl_.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index dc147467829..03e7c563319 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1940,7 +1940,7 @@ mod tests { let out = interpreter .run_python_script_with_envs( "import os; print(os.getenv('PYO3_TEST'))", - [("PYO3_TEST", "42")], + vec![("PYO3_TEST", "42")], ) .expect("failed to run Python script"); assert_eq!(out.trim_end(), "42"); From bc356f23e08f4756233cc7a57deaf5b7197baf41 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Tue, 11 Jan 2022 19:59:33 -0500 Subject: [PATCH 08/11] Fix coverage for pyo3-build-config CI ("full" feature) --- .github/workflows/ci.yml | 2 +- Cargo.toml | 14 +++++++++++++- pyo3-build-config/src/impl_.rs | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 589e3ad3012..6e0cd74a7c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,7 +227,7 @@ jobs: run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml - name: Test build config - run: cargo test --manifest-path=pyo3-build-config/Cargo.toml --all-features + run: cargo test --manifest-path=pyo3-build-config/Cargo.toml - name: Test python examples and tests shell: bash diff --git a/Cargo.toml b/Cargo.toml index 0c83a75c29b..915008e4f14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,19 @@ nightly = [] # Activates all additional features # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. -full = ["macros", "pyproto", "multiple-pymethods", "num-bigint", "num-complex", "hashbrown", "serde", "indexmap", "eyre", "anyhow"] +full = [ + "macros", + "pyproto", + "multiple-pymethods", + "num-bigint", + "num-complex", + "hashbrown", + "serde", + "indexmap", + "eyre", + "anyhow", + "pyo3-build-config/export-config" +] [[bench]] name = "bench_call" diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 03e7c563319..97f5b4c23eb 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1948,7 +1948,7 @@ mod tests { #[test] #[cfg(feature = "export-config")] - fn test_emit_pyo3_configs_round_trip() { + fn test_emit_pyo3_configs_roundtrip() { let interpreter = make_interpreter_config() .expect("could not get InterpreterConfig from installed interpreter"); interpreter.emit_pyo3_cfgs(); From 6e0a20c8cfc92c6d74ab6d356dcf950a9db0bdb1 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Mon, 17 Jan 2022 20:22:19 -0500 Subject: [PATCH 09/11] Mention export-config feature in the guide. --- guide/src/building_and_distribution.md | 2 ++ guide/src/features.md | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 5a3a7ab5944..708436d9cdd 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -50,6 +50,8 @@ If you save the above output config from `PYO3_PRINT_CONFIG` to a file, it is po If your build environment is unusual enough that PyO3's regular configuration detection doesn't work, using a config file like this will give you the flexibility to make PyO3 work for you. To see the full set of options supported, see the documentation for the [`InterpreterConfig` struct](https://docs.rs/pyo3-build-config/{{#PYO3_DOCS_VERSION}}/pyo3_build_config/struct.InterpreterConfig.html). +The `export-config` feature of the `pyo3-build-config` crate also allows these values to be exported for use in other crates, and `pyo3-build-config` offers a funciton for reading them in and constructing an `InterpreterConfig`. + ## Building Python extension modules Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. diff --git a/guide/src/features.md b/guide/src/features.md index 92d04815a10..a4f48083501 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -81,6 +81,14 @@ The `resolve-config` feature of the `pyo3-build-config` crate controls whether t build script automatically resolves a Python interpreter / build configuration. This feature is primarily useful when building PyO3 itself. By default this feature is not enabled, meaning you can freely use `pyo3-build-config` as a standalone library to read or write PyO3 build configuration files or resolve metadata about a Python interpreter. +### `export-config` + +The `export-config` feature of the `pyo3-build-config` crate controls whether +that crate exports the build configuration for use in other crates. This +feature may be useful if building an additional dependency that requires access +to the Python interpreter used to build PyO3 itself. This feature should be +used with caution as it can impact build portability. + ## Optional Dependencies These features enable conversions between Python types and types from other Rust crates, enabling easy access to the rest of the Rust ecosystem. From bbe7a493d611563324b1c186d840dc89094885d9 Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Thu, 17 Mar 2022 10:29:43 -0400 Subject: [PATCH 10/11] Refactor to serialize the whole InterpreterConfig into DEP_PYTHON_PYO3_CONFIG --- Cargo.toml | 1 - guide/src/building_and_distribution.md | 2 - guide/src/features.md | 8 --- pyo3-build-config/Cargo.toml | 3 - pyo3-build-config/src/impl_.rs | 87 ++++++++++---------------- pyo3-build-config/src/lib.rs | 3 +- pyo3-ffi/build.rs | 3 + 7 files changed, 37 insertions(+), 70 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 915008e4f14..bc113de955f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,6 @@ full = [ "indexmap", "eyre", "anyhow", - "pyo3-build-config/export-config" ] [[bench]] diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 708436d9cdd..5a3a7ab5944 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -50,8 +50,6 @@ If you save the above output config from `PYO3_PRINT_CONFIG` to a file, it is po If your build environment is unusual enough that PyO3's regular configuration detection doesn't work, using a config file like this will give you the flexibility to make PyO3 work for you. To see the full set of options supported, see the documentation for the [`InterpreterConfig` struct](https://docs.rs/pyo3-build-config/{{#PYO3_DOCS_VERSION}}/pyo3_build_config/struct.InterpreterConfig.html). -The `export-config` feature of the `pyo3-build-config` crate also allows these values to be exported for use in other crates, and `pyo3-build-config` offers a funciton for reading them in and constructing an `InterpreterConfig`. - ## Building Python extension modules Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. diff --git a/guide/src/features.md b/guide/src/features.md index a4f48083501..92d04815a10 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -81,14 +81,6 @@ The `resolve-config` feature of the `pyo3-build-config` crate controls whether t build script automatically resolves a Python interpreter / build configuration. This feature is primarily useful when building PyO3 itself. By default this feature is not enabled, meaning you can freely use `pyo3-build-config` as a standalone library to read or write PyO3 build configuration files or resolve metadata about a Python interpreter. -### `export-config` - -The `export-config` feature of the `pyo3-build-config` crate controls whether -that crate exports the build configuration for use in other crates. This -feature may be useful if building an additional dependency that requires access -to the Python interpreter used to build PyO3 itself. This feature should be -used with caution as it can impact build portability. - ## Optional Dependencies These features enable conversions between Python types and types from other Rust crates, enabling easy access to the rest of the Rust ecosystem. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 88c1b1df384..2233dd78a4e 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -20,9 +20,6 @@ default = [] # script. If this feature isn't enabled, the build script no-ops. resolve-config = [] -# Export the resolved build config into a file for use by e.g. other build scripts. -export-config = ["resolve-config"] - abi3 = [] abi3-py37 = ["abi3-py38"] abi3-py38 = ["abi3-py39"] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 97f5b4c23eb..13fcf05ad93 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -4,10 +4,11 @@ use std::{ env, ffi::{OsStr, OsString}, fmt::Display, - fs::{self, DirEntry, File}, + fs::{self, DirEntry}, io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, + str, str::FromStr, }; @@ -21,8 +22,6 @@ use crate::{ const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; /// Maximum Python version that can be used as minimum required Python version with abi3. const ABI3_MAX_MINOR: u8 = 9; -/// Name of config file exported by "export-config" feature. -const EXPORT_CONFIG_FILENAME: &str = ".pyo3-export-config"; /// Gets an environment variable owned by cargo. /// @@ -155,25 +154,7 @@ impl InterpreterConfig { } for flag in &self.build_flags.0 { - println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag) - } - - if cfg!(feature = "export-config") { - let output_path = - Path::new(&cargo_env_var("OUT_DIR").unwrap()).join(EXPORT_CONFIG_FILENAME); - if let Ok(config_file) = File::create(&output_path) { - if self.to_writer(config_file).is_err() { - warn!( - "Failed to export build config - this may be a problem for external crates that \ - depend on the pyo3 config." - ); - } else { - println!( - "cargo:PYO3_EXPORT_CONFIG={}", - output_path.canonicalize().unwrap().to_str().unwrap() - ); - } - } + println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag); } } @@ -359,13 +340,13 @@ print("mingw", get_platform().startswith("mingw")) InterpreterConfig::from_reader(reader) } - /// Create an InterpreterConfig generated from pyo3-build-config using the "export-config" - /// feature - #[cfg(feature = "export-config")] - pub fn from_pyo3_export_config() -> Result { - InterpreterConfig::from_path(Path::new( - &cargo_env_var("DEP_PYTHON_PYO3_EXPORT_CONFIG").unwrap(), - )) + #[doc(hidden)] + pub fn from_cargo_link_env() -> Result { + // un-escape any newlines in the exported config + let buf = cargo_env_var("DEP_PYTHON_PYO3_CONFIG") + .unwrap() + .replace("\\n", "\n"); + InterpreterConfig::from_reader(buf.as_bytes()) } #[doc(hidden)] @@ -448,6 +429,19 @@ print("mingw", get_platform().startswith("mingw")) }) } + #[doc(hidden)] + pub fn to_cargo_link_env(&self) -> Result<()> { + let mut buf = Vec::new(); + self.to_writer(&mut buf)?; + // escape newlines in env var + if let Ok(config) = str::from_utf8(&buf) { + println!("cargo:PYO3_CONFIG={}", config.replace('\n', "\\n")); + } else { + bail!("unable to emit interpreter config to link env for downstream use"); + } + Ok(()) + } + #[doc(hidden)] pub fn to_writer(&self, mut writer: impl Write) -> Result<()> { macro_rules! write_line { @@ -500,11 +494,7 @@ print("mingw", get_platform().startswith("mingw")) V: AsRef, { run_python_script_with_envs( - Path::new( - self.executable - .as_ref() - .expect("No interpreter executable!"), - ), + Path::new(self.executable.as_ref().expect("no interpreter executable")), script, envs, ) @@ -512,13 +502,10 @@ print("mingw", get_platform().startswith("mingw")) /// Run a python script using the executable of this InterpreterConfig. pub fn run_python_script(&self, script: &str) -> Result { - run_python_script( - Path::new( - self.executable - .as_ref() - .expect("No interpreter executable!"), - ), + run_python_script_with_envs( + Path::new(self.executable.as_ref().expect("no interpreter executable")), script, + std::iter::empty::<(&str, &str)>(), ) } } @@ -605,6 +592,11 @@ fn is_abi3() -> bool { cargo_env_var("CARGO_FEATURE_ABI3").is_some() } +#[allow(unused)] +pub fn link_env_var_set() -> bool { + cargo_env_var("DEP_PYTHON_PYO3_CONFIG").is_some() +} + #[derive(Debug, PartialEq)] struct TargetInfo { /// The `arch` component of the compilation target triple. @@ -1945,19 +1937,4 @@ mod tests { .expect("failed to run Python script"); assert_eq!(out.trim_end(), "42"); } - - #[test] - #[cfg(feature = "export-config")] - fn test_emit_pyo3_configs_roundtrip() { - let interpreter = make_interpreter_config() - .expect("could not get InterpreterConfig from installed interpreter"); - interpreter.emit_pyo3_cfgs(); - // config path env var is not set during testing of this crate alone, as it relies on the - // pyo3 python links manifest key, so we have to hard-code the same path - let output_path = - Path::new(&cargo_env_var("OUT_DIR").unwrap()).join(EXPORT_CONFIG_FILENAME); - let reimported_interpreter = InterpreterConfig::from_path(output_path) - .expect("could not generated InterpreterConfig from exported config"); - assert_eq!(interpreter, reimported_interpreter); - } } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 8a52fe0dfa5..be1165c9db2 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -61,7 +61,6 @@ fn _add_extension_module_link_args(target_os: &str, mut writer: impl std::io::Wr /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. -#[doc(hidden)] #[cfg(feature = "resolve-config")] pub fn get() -> &'static InterpreterConfig { static CONFIG: OnceCell = OnceCell::new(); @@ -72,6 +71,8 @@ pub fn get() -> &'static InterpreterConfig { Ok(abi3_config()) } else if impl_::cross_compile_env_vars().any() { InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) + } else if impl_::link_env_var_set() { + InterpreterConfig::from_cargo_link_env() } else { InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) } diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index c8d65d5d1c2..98e78c9c4b9 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -69,6 +69,9 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { } } + // serialize the whole interpreter config in DEP_PYTHON_PYO3_CONFIG + interpreter_config.to_cargo_link_env()?; + Ok(()) } From 4a62a62caedfcb338265853aab42890c2c5ec6ff Mon Sep 17 00:00:00 2001 From: Ashley Anderson Date: Mon, 21 Mar 2022 12:55:39 -0400 Subject: [PATCH 11/11] Improvements based on code review: API and documentation. --- CHANGELOG.md | 3 +- pyo3-build-config/src/impl_.rs | 54 ++++++++++++++++++++-------------- pyo3-build-config/src/lib.rs | 8 ++--- pyo3-ffi/build.rs | 2 +- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8c6941b3de..c4505671915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Allow to allow dependent crates to access values from `pyo3-build-config` via cargo link env vars. [#2092](https://github.com/PyO3/pyo3/pull/2092) +- Allow dependent crates to access config values from `pyo3-build-config` via cargo link dep env vars. [#2092](https://github.com/PyO3/pyo3/pull/2092) +- Added methods on `InterpreterConfig` to run Python scripts using the configured executable. [#2092](https://github.com/PyO3/pyo3/pull/2092) ### Fixed diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 13fcf05ad93..f57116a3bc5 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -341,12 +341,9 @@ print("mingw", get_platform().startswith("mingw")) } #[doc(hidden)] - pub fn from_cargo_link_env() -> Result { - // un-escape any newlines in the exported config - let buf = cargo_env_var("DEP_PYTHON_PYO3_CONFIG") - .unwrap() - .replace("\\n", "\n"); - InterpreterConfig::from_reader(buf.as_bytes()) + pub fn from_cargo_dep_env() -> Option> { + cargo_env_var("DEP_PYTHON_PYO3_CONFIG") + .map(|buf| InterpreterConfig::from_reader(buf.replace("\\n", "\n").as_bytes())) } #[doc(hidden)] @@ -430,7 +427,17 @@ print("mingw", get_platform().startswith("mingw")) } #[doc(hidden)] - pub fn to_cargo_link_env(&self) -> Result<()> { + /// Serialize the `InterpreterConfig` and print it to the environment for Cargo to pass along + /// to dependent packages during build time. + /// + /// NB: writing to the cargo environment requires the + /// [`links`](https://doc.rust-lang.org/cargo/reference/build-scripts.html#the-links-manifest-key) + /// manifest key to be set. In this case that means this is called by the `pyo3-ffi` crate and + /// available for dependent package build scripts in `DEP_PYTHON_PYO3_CONFIG`. See + /// documentation for the + /// [`DEP__`](https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts) + /// environment variable. + pub fn to_cargo_dep_env(&self) -> Result<()> { let mut buf = Vec::new(); self.to_writer(&mut buf)?; // escape newlines in env var @@ -485,8 +492,25 @@ print("mingw", get_platform().startswith("mingw")) Ok(()) } - /// Run a python script using the executable of this InterpreterConfig with additional + /// Run a python script using the [`InterpreterConfig::executable`]. + /// + /// # Panics + /// + /// This function will panic if the [`executable`](InterpreterConfig::executable) is `None`. + pub fn run_python_script(&self, script: &str) -> Result { + run_python_script_with_envs( + Path::new(self.executable.as_ref().expect("no interpreter executable")), + script, + std::iter::empty::<(&str, &str)>(), + ) + } + + /// Run a python script using the [`InterpreterConfig::executable`] with additional /// environment variables (e.g. PYTHONPATH) set. + /// + /// # Panics + /// + /// This function will panic if the [`executable`](InterpreterConfig::executable) is `None`. pub fn run_python_script_with_envs(&self, script: &str, envs: I) -> Result where I: IntoIterator, @@ -499,15 +523,6 @@ print("mingw", get_platform().startswith("mingw")) envs, ) } - - /// Run a python script using the executable of this InterpreterConfig. - pub fn run_python_script(&self, script: &str) -> Result { - run_python_script_with_envs( - Path::new(self.executable.as_ref().expect("no interpreter executable")), - script, - std::iter::empty::<(&str, &str)>(), - ) - } } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -592,11 +607,6 @@ fn is_abi3() -> bool { cargo_env_var("CARGO_FEATURE_ABI3").is_some() } -#[allow(unused)] -pub fn link_env_var_set() -> bool { - cargo_env_var("DEP_PYTHON_PYO3_CONFIG").is_some() -} - #[derive(Debug, PartialEq)] struct TargetInfo { /// The `arch` component of the compilation target triple. diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index be1165c9db2..814ff1da811 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -65,18 +65,18 @@ fn _add_extension_module_link_args(target_os: &str, mut writer: impl std::io::Wr pub fn get() -> &'static InterpreterConfig { static CONFIG: OnceCell = OnceCell::new(); CONFIG.get_or_init(|| { - if !CONFIG_FILE.is_empty() { + if let Some(interpreter_config) = InterpreterConfig::from_cargo_dep_env() { + interpreter_config + } else if !CONFIG_FILE.is_empty() { InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) } else if !ABI3_CONFIG.is_empty() { Ok(abi3_config()) } else if impl_::cross_compile_env_vars().any() { InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH) - } else if impl_::link_env_var_set() { - InterpreterConfig::from_cargo_link_env() } else { InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) } - .expect("failed to parse PyO3 config file") + .expect("failed to parse PyO3 config") }) } diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 98e78c9c4b9..ddc60e62e5e 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -70,7 +70,7 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { } // serialize the whole interpreter config in DEP_PYTHON_PYO3_CONFIG - interpreter_config.to_cargo_link_env()?; + interpreter_config.to_cargo_dep_env()?; Ok(()) }