Releasing 1.8.2 [rebase & merge]

.actions/assistant.py

@@ -1,8 +1,9 @@
 import os
 import re
+import shutil
 from itertools import chain
+from os.path import dirname, isfile
 from pathlib import Path
-from pprint import pprint
 from typing import Dict, List, Optional, Sequence, Tuple
 
 import pkg_resources
@@ -65,6 +66,7 @@ def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]]) -> List[s
 def copy_replace_imports(
     source_dir: str, source_imports: List[str], target_imports: List[str], target_dir: Optional[str] = None
 ) -> None:
+    """Copy package content with import adjustments."""
     print(f"Replacing imports: {locals()}")
     assert len(source_imports) == len(target_imports), (
         "source and target imports must have the same length, "
@@ -75,19 +77,27 @@ def copy_replace_imports(
 
     ls = _retrieve_files(source_dir)
     for fp in ls:
-        if fp.endswith(".py") or not fp.endswith(".pyc"):
-            with open(fp, encoding="utf-8") as fo:
-                try:
-                    lines = fo.readlines()
-                except UnicodeDecodeError:
-                    # a binary file, skip
-                    print(f"Skipped replacing imports for {fp}")
-                    continue
-            lines = _replace_imports(lines, list(zip(source_imports, target_imports)))
-            fp_new = fp.replace(source_dir, target_dir)
-            os.makedirs(os.path.dirname(fp_new), exist_ok=True)
-            with open(fp_new, "w", encoding="utf-8") as fo:
-                fo.writelines(lines)
+        fp_new = fp.replace(source_dir, target_dir)
+        _, ext = os.path.splitext(fp)
+        if ext in (".png", ".jpg", ".ico"):
+            os.makedirs(dirname(fp_new), exist_ok=True)
+            if not isfile(fp_new):
+                shutil.copy(fp, fp_new)
+            continue
+        elif ext in (".pyc",):
+            continue
+        # Try to parse everything else
+        with open(fp, encoding="utf-8") as fo:
+            try:
+                lines = fo.readlines()
+            except UnicodeDecodeError:
+                # a binary file, skip
+                print(f"Skipped replacing imports for {fp}")
+                continue
+        lines = _replace_imports(lines, list(zip(source_imports, target_imports)))
+        os.makedirs(os.path.dirname(fp_new), exist_ok=True)
+        with open(fp_new, "w", encoding="utf-8") as fo:
+            fo.writelines(lines)
 
 
 def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None:
@@ -129,7 +139,7 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
             req = list(pkg_resources.parse_requirements(ln_))[0]
             if req.name not in packages:
                 final.append(line)
-        pprint(final)
+        print(final)
         path.write_text("\n".join(final))
 
     @staticmethod
@@ -147,7 +157,7 @@ def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL
     def copy_replace_imports(
         source_dir: str, source_import: str, target_import: str, target_dir: Optional[str] = None
     ) -> None:
-        """Recursively replace imports in given folder."""
+        """Copy package content with import adjustments."""
         source_imports = source_import.strip().split(",")
         target_imports = target_import.strip().split(",")
         copy_replace_imports(source_dir, source_imports, target_imports, target_dir=target_dir)

.azure/app-cloud-e2e.yml

@@ -27,12 +27,17 @@ pr:
       - "release/*"
   paths:
     include:
-    - ".azure/app-cloud-e2e.yml"
-    - "requirements/app/**"
-    - "src/lightning_app/**"
-    - "examples/app_*"
-    - "tests/tests_app_examples/**"
-    - ".actions/**"
+      - ".azure/app-cloud-e2e.yml"
+      - "requirements/app/**"
+      - "src/lightning_app/**"
+      - "tests/tests_app/**"
+      - "examples/app_*/**"  # some tests_app tests call examples files
+      - "tests/tests_app_examples/**"
+      - "setup.py"
+      - ".actions/**"
+      - "!requirements/app/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 # variables are automatically exported as environment variables so this will override pip's default cache dir
 variables:

.azure/gpu-benchmark.yml

@@ -21,6 +21,11 @@ pr:
   paths:
     include:
       - ".azure/gpu-benchmark.yml"
+      - "tests/tests_pytorch/benchmarks/**"
+      - "requirements/pytorch/**"
+      - "!requirements/pytorch/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 schedules:
   - cron: "0 0 * * *" # At the end of every day
@@ -37,7 +42,7 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.13-cuda11.6.1"
       options: "--gpus=all --shm-size=32g"
     workspace:
       clean: all
@@ -47,18 +52,41 @@ jobs:
     - bash: |
         echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
         cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$cuda_ver"
         echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
       displayName: 'set env. vars'
 
     - bash: |
-        pip install -e .[strategies] --find-links ${TORCH_URL}
+        echo $CUDA_VISIBLE_DEVICES
+        echo $TORCH_URL
+        lspci | egrep 'VGA|3D'
+        whereis nvidia
+        nvidia-smi
+        which python && which pip
+        python --version
+        pip --version
         pip list
+      displayName: 'Image info & NVIDIA'
+
+    - bash: |
+        python .actions/assistant.py requirements_prune_pkgs --packages [horovod,bagua,colossalai] --req_files [requirements/pytorch/strategies.txt]
+
+        PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
+        python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${PYTORCH_VERSION}
+      displayName: 'Adjust dependencies'
+
+    - bash: pip install -e .[dev,strategies,examples] --find-links ${TORCH_URL}
       env:
-        PACKAGE_NAME: pytorch
-        FREEZE_REQUIREMENTS: 1
+        PACKAGE_NAME: "pytorch"
+        FREEZE_REQUIREMENTS: "1"
       displayName: 'Install package'
 
+    - bash: |
+        set -e
+        pip list
+        python requirements/collect_env_details.py
+        python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
+      displayName: 'Env details'
+
     - bash: python -m pytest benchmarks -v --durations=0
       env:
         PL_RUNNING_BENCHMARKS: "1"

.azure/gpu-tests-lite.yml

@@ -21,12 +21,18 @@ pr:
   paths:
     include:
       - ".azure/gpu-tests-lite.yml"
+      - "examples/lite/**"
+      - "examples/run_lite_examples.sh"
+      - "tests/tests_lite/run_standalone_*.sh"
+      - "tests/tests_pytorch/run_standalone_tests.sh"  # used by Lite through a symlink
       - "requirements/lite/**"
       - "src/lightning_lite/**"
       - "tests/tests_lite/**"
-      - "tests/tests_pytorch/run_standalone_tests.sh"
-      - "tests/tests_lite/run_standalone_tests.sh"  # a symlink to the one above
+      - "setup.cfg"  # includes pytest config
       - ".actions/**"
+      - "!requirements/lite/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 jobs:
   - job: testing
@@ -38,7 +44,7 @@ jobs:
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.13-cuda11.6.1"
       # default shm size is 64m. Increase it to avoid:
       # 'Error while creating shared memory: unhandled system error, NCCL version 2.7.8'
       options: "--gpus=all --shm-size=2gb"
@@ -48,6 +54,14 @@ jobs:
 
     steps:
     - bash: |
+        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+        cuda_ver=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
+        echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${cuda_ver}/torch_stable.html"
+      displayName: 'set env. vars'
+
+    - bash: |
+        echo $CUDA_VISIBLE_DEVICES
+        echo $TORCH_URL
         lspci | egrep 'VGA|3D'
         whereis nvidia
         nvidia-smi
@@ -58,22 +72,21 @@ jobs:
       displayName: 'Image info & NVIDIA'
 
     - bash: |
-        echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
-      displayName: 'set visible devices'
+        PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
+        python ./requirements/pytorch/adjust-versions.py requirements/lite/base.txt ${PYTORCH_VERSION}
+        python ./requirements/pytorch/adjust-versions.py requirements/lite/examples.txt ${PYTORCH_VERSION}
+      displayName: 'Adjust dependencies'
 
     - bash: |
-        set -e
-        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        pip install -e .[dev,strategies] --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
-        pip list
+        pip install -e .[dev,strategies,examples] --find-links ${TORCH_URL}
       env:
         PACKAGE_NAME: "lite"
         FREEZE_REQUIREMENTS: "1"
       displayName: 'Install package & dependencies'
 
     - bash: |
         set -e
-        echo $CUDA_VISIBLE_DEVICES
+        pip list
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
       displayName: 'Env details'

.azure/gpu-tests-pytorch.yml

@@ -37,14 +37,20 @@ pr:
       - "requirements/lite/**"
       - "src/lightning_lite/**"
       - ".actions/**"
+      - "!requirements/**/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 jobs:
   - job: testing
     strategy:
       matrix:
-        # TODO: package parametrization
-        'PyTorch - stable':
+        'PyTorch & strategies':  # this uses torch 1.12 as not all strategies support 1.13 yet
           image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
+          scope: "strategies"
+        'PyTorch - latest':
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.13-cuda11.6.1"
+          scope: ""
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "80"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -93,11 +99,11 @@ jobs:
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION}
       displayName: 'Adjust dependencies'
 
-    - bash: pip install -e .[strategies] -r requirements/pytorch/devel.txt -r requirements/pytorch/examples.txt --find-links ${TORCH_URL}
+    - bash: pip install -e .[dev,examples] --find-links ${TORCH_URL}
       env:
         PACKAGE_NAME: "pytorch"
         FREEZE_REQUIREMENTS: "1"
-      displayName: 'Install package'
+      displayName: 'Install package & extras'
 
     - bash: |
         set -e
@@ -109,14 +115,17 @@ jobs:
         CUDA_VERSION_COLOSSALAI=$(python -c "print([ver for ver in [11.3, 11.1] if $CUDA_VERSION_MM_COLOSSALAI >= ver][0])")
         pip install "colossalai==0.1.10+torch${PYTORCH_VERSION_COLOSSALAI}cu${CUDA_VERSION_COLOSSALAI}" --find-links https://release.colossalai.org
 
-        pip list
-      displayName: 'Install dependencies'
+        pip install -r requirements/pytorch/strategies.txt --find-links ${TORCH_URL}
+
+        python requirements/pytorch/check-avail-strategies.py
+      condition: eq(variables['scope'], 'strategies')
+      displayName: 'Install strategies'
 
     - bash: |
         set -e
+        pip list
         python requirements/collect_env_details.py
         python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
-        python requirements/pytorch/check-avail-strategies.py
         python requirements/pytorch/check-avail-extras.py
       displayName: 'Env details'
 

.azure/hpu-tests.yml

@@ -26,6 +26,9 @@ pr:
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
       - ".actions/**"
+      - "!requirements/**/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 jobs:
   - job: testing

.azure/ipu-tests.yml

@@ -23,6 +23,9 @@ pr:
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
       - ".actions/**"
+      - "!requirements/**/docs.txt"
+      - "!*.md"
+      - "!**/*.md"
 
 variables:
   - name: poplar_sdk