From 39429810e6560433c5dc60c72471609fb911805b Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Fri, 18 Nov 2022 14:09:21 +0000 Subject: [PATCH] Add platform markers for linux only extra_install_requires (#88826) Fixes #88049 https://github.com/pytorch/pytorch/pull/85097 added new extra dependencies on `nvidia-*`. They are linux (GPU) only packages, but were not marked as such, causing issues installing pytorch 1.13 via Poetry (and possibly other tools that follow PyPI's metadata API) on non-Linux systems. This "fixes" the issue by adding the `; platform_system = 'Linux'` marker on these dependencies, but the main problem of different metadata for different wheels is a [somewhat larger issue](https://github.com/pytorch/pytorch/issues/88049#issuecomment-1302555269). https://github.com/pytorch/pytorch/pull/85097 used `;` as a delimiter for splitting the different deps, but that is the delimiter used in markers, so I changed to split on `|`. Pull Request resolved: https://github.com/pytorch/pytorch/pull/88826 Approved by: https://github.com/neersighted, https://github.com/lalmei, https://github.com/malfet --- .github/scripts/generate_binary_build_matrix.py | 6 +++--- .../generated-linux-binary-manywheel-nightly.yml | 10 +++++----- setup.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/scripts/generate_binary_build_matrix.py b/.github/scripts/generate_binary_build_matrix.py index 54949ff27bb1bb7..4031ee9aacca675 100644 --- a/.github/scripts/generate_binary_build_matrix.py +++ b/.github/scripts/generate_binary_build_matrix.py @@ -219,9 +219,9 @@ def generate_wheels_matrix(os: str, "container_image": WHEEL_CONTAINER_IMAGES[arch_version], "package_type": package_type, "pytorch_extra_install_requirements": - "nvidia-cuda-runtime-cu11;" - "nvidia-cudnn-cu11==8.5.0.96;" - "nvidia-cublas-cu11==11.10.3.66", + "nvidia-cuda-runtime-cu11; platform_system == 'Linux' | " + "nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | " + "nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux'", "build_name": f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}-with-pypi-cudnn" .replace( diff --git a/.github/workflows/generated-linux-binary-manywheel-nightly.yml b/.github/workflows/generated-linux-binary-manywheel-nightly.yml index 928ad3e0ca13eaa..5fbab85a1d32ec9 100644 --- a/.github/workflows/generated-linux-binary-manywheel-nightly.yml +++ b/.github/workflows/generated-linux-binary-manywheel-nightly.yml @@ -169,7 +169,7 @@ jobs: DESIRED_PYTHON: "3.7" build_name: manywheel-py3_7-cuda11_7-with-pypi-cudnn build_environment: linux-binary-manywheel - PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11;nvidia-cudnn-cu11==8.5.0.96;nvidia-cublas-cu11==11.10.3.66 + PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11; platform_system == 'Linux' | nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux' secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -685,7 +685,7 @@ jobs: DESIRED_PYTHON: "3.8" build_name: manywheel-py3_8-cuda11_7-with-pypi-cudnn build_environment: linux-binary-manywheel - PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11;nvidia-cudnn-cu11==8.5.0.96;nvidia-cublas-cu11==11.10.3.66 + PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11; platform_system == 'Linux' | nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux' secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -1201,7 +1201,7 @@ jobs: DESIRED_PYTHON: "3.9" build_name: manywheel-py3_9-cuda11_7-with-pypi-cudnn build_environment: linux-binary-manywheel - PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11;nvidia-cudnn-cu11==8.5.0.96;nvidia-cublas-cu11==11.10.3.66 + PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11; platform_system == 'Linux' | nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux' secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -1717,7 +1717,7 @@ jobs: DESIRED_PYTHON: "3.10" build_name: manywheel-py3_10-cuda11_7-with-pypi-cudnn build_environment: linux-binary-manywheel - PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11;nvidia-cudnn-cu11==8.5.0.96;nvidia-cublas-cu11==11.10.3.66 + PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11; platform_system == 'Linux' | nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux' secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -2233,7 +2233,7 @@ jobs: DESIRED_PYTHON: "3.11" build_name: manywheel-py3_11-cuda11_7-with-pypi-cudnn build_environment: linux-binary-manywheel - PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11;nvidia-cudnn-cu11==8.5.0.96;nvidia-cublas-cu11==11.10.3.66 + PYTORCH_EXTRA_INSTALL_REQUIREMENTS: nvidia-cuda-runtime-cu11; platform_system == 'Linux' | nvidia-cudnn-cu11==8.5.0.96; platform_system == 'Linux' | nvidia-cublas-cu11==11.10.3.66; platform_system == 'Linux' secrets: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/setup.py b/setup.py index b618b4571fbd91f..40defe2424befbf 100644 --- a/setup.py +++ b/setup.py @@ -848,7 +848,7 @@ def configure_extension_build(): pytorch_extra_install_requirements = os.getenv("PYTORCH_EXTRA_INSTALL_REQUIREMENTS", "") if pytorch_extra_install_requirements: report(f"pytorch_extra_install_requirements: {pytorch_extra_install_requirements}") - extra_install_requires += pytorch_extra_install_requirements.split(";") + extra_install_requires += pytorch_extra_install_requirements.split("|") # Cross-compile for M1