release-pypi.yml

name: PyPI

# https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:
  push:
    branches: [master, "release/*"]
  release:
    types: [published]

defaults:
  run:
    shell: bash

jobs:
  init:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - run: |
        mkdir dist && touch dist/.placeholder
    - uses: actions/upload-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist

  build-packages:
    needs: init
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: true
      max-parallel: 1  # run sequential to prevent download/upload collisions
      matrix:
        pkg-name: ["lightning", "app", "lite", "pytorch"]
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - uses: actions/setup-python@v4
      with:
        python-version: 3.9

    - name: Install dependencies
      run: pip install -U setuptools wheel

    - name: Build packages
      env:
        PACKAGE_NAME: ${{ matrix.pkg-name }}
      run: |
        python setup.py sdist bdist_wheel
        ls -lh dist/

    - uses: actions/upload-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist

  upload-packages:
    runs-on: ubuntu-20.04
    needs: build-packages
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - name: Upload to release
      uses: AButler/upload-release-assets@v2.0
      with:
        files: 'dist/*'
        repo-token: ${{ secrets.GITHUB_TOKEN }}

  release-version:
    runs-on: ubuntu-20.04
    outputs:
      tag: ${{ steps.lai-package.outputs.version }}
    steps:
    - uses: actions/checkout@v3
    - name: install Package
      env:
        PACKAGE_NAME: "lightning"
      run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
    - name: package Version
      id: lai-package
      run: python -c "import lightning as L; print(f'version={L.__version__}')" >> $GITHUB_OUTPUT

  signaling:
    runs-on: ubuntu-20.04
    needs: [release-version]
    env:
      TAG: ${{ needs.release-version.outputs.tag }}
    steps:
    - uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - uses: actions/checkout@v3
      with:
        repository: gridai/base-images
        token: ${{ secrets.PAT_GHOST }}
        ref: main
    - uses: fregante/setup-git-token@v1
      with:
        token: ${{ secrets.PAT_GHOST }}
        name: PL Ghost
        email: pl-github@grid.ai
    - name: Update lightning version
      run: |
        import json, os
        with open("versions.json") as fo:
            vers = json.load(fo)
        vers["lightning_version"] = os.getenv('TAG')
        with open("versions.json", "w") as fw:
            json.dump(vers, fw)
      shell: python
    - name: GIT Commit
      run: |
        git add versions.json
        git commit -m "bumping lightning version -> ${TAG}"
        cat versions.json
    - name: GIT Push
      run: |
        git status
        git push

  waiting:
    # TODO: replace with back signal from build images/ loop checking for a specific branch?
    runs-on: ubuntu-20.04
    needs: [release-version, signaling]
    env:
      TAG: ${{ needs.release-version.outputs.tag }}
    timeout-minutes: 90
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - run: pip install gitpython
    - name: Delay releasing
      run: |
        import git, os, time
        repo = git.Repo('.')
        branch = f"origin/builds/{os.getenv('TAG')}"
        while True:
          remote_refs = [b.name for b in repo.remote().refs]
          if branch in remote_refs:
            break
          time.sleep(60)
      shell: python

  pre-publish-packages:
    runs-on: ubuntu-20.04
    needs: build-packages
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - run: mkdir pypi/
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*app*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_APP }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*lite*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LITE }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*pytorch*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_PYTORCH }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*"
        pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LAI }}

  publish-packages:
    runs-on: ubuntu-20.04
    needs: waiting
    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v3
      with:
        name: dist-packages-${{ github.sha }}
        path: dist
    - run: ls -lh dist/
    - run: mkdir pypi/
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*app*"
        pypi-token: ${{ secrets.PYPI_TOKEN_APP }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*lite*"
        pypi-token: ${{ secrets.PYPI_TOKEN_LITE }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*pytorch*"
        pypi-token: ${{ secrets.PYPI_TOKEN_PYTORCH }}
    - uses: ./.github/actions/pkg-publish
      with:
        pkg-pattern: "*"
        pypi-token: ${{ secrets.PYPI_TOKEN_LAI }}

  legacy-checkpoints:
    needs: [build-packages]
    uses: ./.github/workflows/legacy-checkpoints.yml
    with:
      push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
      create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }}