CI: clean install & share pkg build (#15986)

* abstract pkg build * share ci * syntax * Checkgroup * folders * whl 1st * doctest Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com> (cherry picked from commit 18a4638)
Lightning-AI · Dec 14, 2022 · dc96640 · dc96640
1 parent 6fddd82
commit dc96640
Show file tree

Hide file tree

Showing 21 changed files with 182 additions and 108 deletions.
diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml
@@ -5,7 +5,6 @@ inputs:
   pkg-name:
     description: package name inside lightning.*
     required: true
-    default: ""
   nb-dirs:
     description: nb of packages in the wrap/distribution
     required: false

diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml
@@ -2,6 +2,9 @@ name: Install and validate the package
 description: Install and validate the package
 
 inputs:
+  pkg-folder:
+    description: Define folder with packages
+    required: true
   pkg-name:
     description: Package name to import
     required: true
@@ -14,22 +17,26 @@ runs:
   using: "composite"
   steps:
   - name: Choose package import
+    working-directory: ${{ inputs.pkg-folder }}
     run: |
-      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])" >> $GITHUB_ENV
+      ls -l
+      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning'}.get('${{matrix.pkg-name}}', 'lightning'))" >> $GITHUB_ENV
+      python -c "import glob ; ls = glob.glob('*.tar.gz') ; print('PKG_SOURCE=' + ls[0])" >> $GITHUB_ENV
+      python -c "import glob ; ls = glob.glob('*.whl') ; print('PKG_WHEEL=' + ls[0])" >> $GITHUB_ENV
     shell: bash
 
-  - name: Install package - archive
-    working-directory: ./dist
+  - name: Install package - wheel
+    working-directory: ${{ inputs.pkg-folder }}
     run: |
-      pip install *.tar.gz ${{ inputs.pip-flags }}
+      pip install ${PKG_WHEEL} ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
 
-  - name: Install package - wheel
-    working-directory: ./dist
+  - name: Install package - archive
+    working-directory: ${{ inputs.pkg-folder }}
     run: |
-      pip install *.whl ${{ inputs.pip-flags }}
+      pip install ${PKG_SOURCE} ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
diff --git a/.github/actions/pkg-publish/action.yml b/.github/actions/pkg-publish/action.yml
@@ -2,8 +2,8 @@ name: Publish package
 description: publishing whl and src to PyPI
 
 inputs:
-  pkg-pattern:
-    description: what file pattern is searched in folder, so for example `*_app*`
+  pkg-folder:
+    description: define folder with packages
     required: true
   pypi-test-token:
     description: login token for PyPI
@@ -18,10 +18,7 @@ runs:
   using: "composite"
   steps:
 
-  - name: filter packages
-    run: |
-      mv dist/${{ inputs.pkg-pattern }} pypi/
-      ls -l pypi/
+  - run: ls -lh ${{ inputs.pkg-folder }}
     shell: bash
 
   # We do this, since failures on test.pypi aren't that bad
@@ -32,7 +29,7 @@ runs:
       user: __token__
       password: ${{ inputs.pypi-test-token }}
       repository_url: https://test.pypi.org/legacy/
-      packages_dir: pypi/
+      packages_dir: ${{ inputs.pkg-folder }}
       verbose: true
 
   - name: Publish distribution 📦 to PyPI
@@ -41,9 +38,5 @@ runs:
     with:
       user: __token__
       password: ${{ inputs.pypi-token }}
-      packages_dir: pypi/
+      packages_dir: ${{ inputs.pkg-folder }}
       verbose: true
-
-  - name: filter packages
-    run: rm pypi/*
-    shell: bash
diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -312,6 +312,9 @@ subprojects:
   - id: "install"
     paths:
       - ".actions/**"
+      - ".github/actions/pkg-check/*"
+      - ".github/actions/pkg-install/*"
+      - ".github/workflows/_build-packages.yml"
       - ".github/workflows/ci-pkg-install.yml"
       - "setup.py"
       - "src/**"

diff --git a/.github/workflows/_build-packages.yml b/.github/workflows/_build-packages.yml
@@ -0,0 +1,65 @@
+name: Building packages
+
+on:
+  workflow_call:
+    inputs:
+      artifact-name:
+        description: 'Unique name for collecting artifacts'
+        required: true
+        type: string
+      pkg-names:
+        description: 'list package names to be build in json format'
+        required: false
+        type: string
+        default: |
+          ["lightning", "app", "lite", "pytorch"]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  init:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - run: |
+        mkdir dist && touch dist/.placeholder
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: dist
+
+
+  build-packages:
+    needs: init
+    runs-on: ubuntu-20.04
+    strategy:
+      max-parallel: 1  # run sequential to prevent download/upload collisions
+      matrix:
+        pkg-name: ${{ fromJSON(inputs.pkg-names) }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: pypi
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
+    - uses: ./.github/actions/pkg-check
+      with:
+        pkg-name: ${{ matrix.pkg-name }}
+        nb-dirs: ${{ env.NB_DIRS }}
+
+    - run: |
+        mkdir pypi/${{ matrix.pkg-name }}
+        cp dist/* pypi/${{ matrix.pkg-name }}/
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: pypi
diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml
@@ -93,12 +93,16 @@ jobs:
 
       - name: Adjust tests
         if: ${{ matrix.pkg-name == 'lightning' }}
-        run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
+        run: |
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning_app" --target_import="lightning.app"
 
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" \
+            --source_import="lightning.app,lightning" \
+            --target_import="lightning_app,lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV

diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml
@@ -99,12 +99,17 @@ jobs:
 
       - name: Adjust tests
         if: ${{ matrix.pkg-name == 'lightning' }}
-        run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app,lightning_lite,pytorch_lightning" --target_import="lightning.app,lightning.lite,lightning.pytorch"
+        run: |
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning_app,lightning_lite,pytorch_lightning" \
+            --target_import="lightning.app,lightning.lite,lightning.pytorch"
 
       - name: Adjust examples
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
-          python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app,lightning" --target_import="lightning_app,lightning_app"
+          python .actions/assistant.py copy_replace_imports --source_dir="./examples" \
+            --source_import="lightning.app,lightning" \
+            --target_import="lightning_app,lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV

diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml
@@ -114,7 +114,9 @@ jobs:
 
     - name: Adjust tests
       if: ${{ matrix.pkg-name == 'lightning' }}
-      run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_lite" --target_import="lightning.lite"
+      run: |
+        python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+          --source_import="lightning_lite" --target_import="lightning.lite"
 
     - name: Testing Warnings
       # the stacklevel can only be set on >=3.7

diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml
@@ -9,6 +9,9 @@ on:
     types: [opened, reopened, ready_for_review, synchronize]  # added `ready_for_review` since draft is skipped
     paths:
       - ".actions/**"
+      - ".github/actions/pkg-check/*"
+      - ".github/actions/pkg-install/*"
+      - ".github/workflows/_build-packages.yml"
       - ".github/workflows/ci-pkg-install.yml"
       - "setup.py"
       - "src/**"
@@ -28,49 +31,71 @@ defaults:
 
 jobs:
 
+  build-packages:
+    uses: ./.github/workflows/_build-packages.yml
+    with:
+      artifact-name: dist-packages-${{ github.sha }}
+
   install-pkg:
+    needs: build-packages
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04, macOS-12, windows-2022]
         pkg-name: ["app", "lite", "pytorch", "lightning"]
         python-version: ["3.7" , "3.10"]
+        # TODO: add also install from source
     steps:
       - uses: actions/checkout@v3
-
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: DocTests actions
-        working-directory: .actions/
-        run: |
-          pip install -q pytest
-          python -m pytest setup_tools.py
-
-      - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
-
-      - uses: ./.github/actions/pkg-check
+      - uses: actions/download-artifact@v3
         with:
-          pkg-name: ${{ matrix.pkg-name }}
-          nb-dirs: ${{ env.NB_DIRS }}
+          name: dist-packages-${{ github.sha }}
+          path: dist
 
+      - run: |
+          python -c "print('PKG_DIR=' + {'notset': 'lightning'}.get('${{matrix.pkg-name}}', '${{matrix.pkg-name}}'))" >> $GITHUB_ENV
       - uses: ./.github/actions/pkg-install
         with:
+          pkg-folder: dist/${{ env.PKG_DIR }}
           pkg-name: ${{ matrix.pkg-name }}
 
-      - name: Run CLI
-        # todo: add testing for `lightning_app`
-        if: ${{ matrix.pkg-name == 'lightning' }}
+      - name: Run CLI (via python)
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
         run: python -m lightning --version
+      - name: Run CLI (direct bash)
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'app' }}
+        run: lightning --version
 
+      - name: Adjust code for Lit
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
+        run: |
+          pip install -q -r .actions/requirements.txt
+          python .actions/assistant.py copy_replace_imports --source_dir="./src" \
+            --source_import="pytorch_lightning,lightning_lite,lightning_app" \
+            --target_import="lightning.pytorch,lightning.lite,lightning.app"
+          rm -rf src/lightning
+      - name: Rename src folders
+        working-directory: src/
+        run: |
+          mv pytorch_lightning pl
+          mv lightning_lite lit_lite
+          mv lightning_app lit_app
+
+      - name: DocTests actions
+        working-directory: .actions/
+        run: |
+          pip install -q pytest
+          python -m pytest setup_tools.py
       - name: DocTest package
         env:
           LIGHTING_TESTING: 1  # path for require wrapper
           PY_IGNORE_IMPORTMISMATCH: 1
         run: |
           pip install -q "pytest-doctestplus>=0.9.0"
           pip list
-          PKG_NAME=$(python -c "print({'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning', 'lightning': 'lightning'}['${{matrix.pkg-name}}'])")
+          PKG_NAME=$(python -c "print({'app': 'lit_app', 'lite': 'lit_lite', 'pytorch': 'pl'}.get('${{matrix.pkg-name}}', ''))")
           python -m pytest src/${PKG_NAME} --ignore-glob="**/cli/*-template/**"
diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml
@@ -169,7 +169,10 @@ jobs:
 
     - name: Adjust tests
       if: ${{ matrix.pkg-name == 'lightning' }}
-      run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite"
+      run: |
+        python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+          --source_import="pytorch_lightning,lightning_lite" \
+          --target_import="lightning.pytorch,lightning.lite"
 
     - name: Testing Warnings
       # the stacklevel can only be set on >=3.7

diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml
@@ -83,7 +83,9 @@ jobs:
         if: ${{ matrix.pkg-name == 'app' }}
         run: |
           pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py copy_replace_imports --source_dir="./docs" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite"
+          python .actions/assistant.py copy_replace_imports --source_dir="./docs" \
+            --source_import="pytorch_lightning,lightning_lite" \
+            --target_import="lightning.pytorch,lightning.lite"
 
       - name: Install this package
         env: