release-pypi.yml

name: PyPI

# https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:
  push:
    branches: [master, "release/*"]
  release:
    types: [published]

defaults:
  run:
    shell: bash

jobs:

  init:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - run: |
        mkdir dist && touch dist/.placeholder
    - uses: actions/upload-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist


  build-packages:
    needs: init
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: true
      max-parallel: 1  # run sequential to prevent download/upload collisions
      matrix:
        pkg-name: ["lightning", "app", "lite", "pytorch"]
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Install dependencies
      run: pip install -U setuptools wheel
    - name: Build packages
      env:
        PACKAGE_NAME: ${{ matrix.pkg-name }}
      run: |
        python setup.py sdist bdist_wheel
        ls -lh dist/
    - uses: actions/upload-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist


  upload-packages:
    runs-on: ubuntu-20.04
    needs: build-packages
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - name: Upload to release
      uses: AButler/upload-release-assets@v2.0
      with:
        files: 'dist/*'
        repo-token: ${{ secrets.GITHUB_TOKEN }}


  release-version:
    runs-on: ubuntu-20.04
    outputs:
      tag: ${{ steps.lai-package.outputs.version }}
    steps:
    - uses: actions/checkout@v3
    - name: install Package
      env:
        PACKAGE_NAME: "lightning"
      run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
    - name: package Version
      id: lai-package
      run: python -c "import lightning as L; print(f'version={L.__version__}')" >> $GITHUB_OUTPUT


  signaling:
    runs-on: ubuntu-20.04
    needs: [release-version]
    env:
      TAG: ${{ needs.release-version.outputs.tag }}
    steps:
    - uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - uses: actions/checkout@v3
      with:
        repository: gridai/base-images
        token: ${{ secrets.PAT_GHOST }}
    - name: Update lightning version
      run: |
        import json, os
        with open("versions.json") as fo:
            vers = json.load(fo)
        vers["lightning_version"] = os.getenv('TAG')
        with open("versions.json", "w") as fw:
            json.dump(vers, fw)
      shell: python
    - run: cat versions.json
    - name: GIT commit & push
      env:
        BRANCH_NAME: "trigger/lightning-${{ env.TAG }}"
      run: |
        git config --global user.name "PL Ghost"
        git config --global user.email pl-github@grid.ai
        git checkout -b ${BRANCH_NAME}
        git add versions.json
        git status
        git commit -m "bumping lightning version -> ${TAG}"
        git push -u origin ${BRANCH_NAME} -f


  waiting:
    runs-on: ubuntu-20.04
    needs: [release-version, signaling]
    env:
      TAG: ${{ needs.release-version.outputs.tag }}
    timeout-minutes: 90
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - run: pip install gitpython
    - name: Delay releasing
      run: |
        import git, os, time
        repo = git.Repo('.')
        branch = f"origin/builds/{os.getenv('TAG')}"
        while True:
          remote_refs = [b.name for b in repo.remote().refs]
          if branch in remote_refs:
            break
          time.sleep(60)
      shell: python


  pre-publish-packages:
    runs-on: ubuntu-20.04
    needs: build-packages
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - run: mkdir pypi/
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*app*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_APP }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*lite*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LITE }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*pytorch*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_PYTORCH }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LAI }}


  publish-packages:
    runs-on: ubuntu-20.04
    needs: [build-packages, waiting]
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - run: mkdir pypi/
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*app*"
        pypi-token: ${{ secrets.PYPI_TOKEN_APP }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*lite*"
        pypi-token: ${{ secrets.PYPI_TOKEN_LITE }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*pytorch*"
        pypi-token: ${{ secrets.PYPI_TOKEN_PYTORCH }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*"
        pypi-token: ${{ secrets.PYPI_TOKEN_LAI }}


  legacy-checkpoints:
    needs: [build-packages]
    uses: ./.github/workflows/legacy-checkpoints.yml
    with:
      push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
      create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }}