-
Notifications
You must be signed in to change notification settings - Fork 3.3k
139 lines (123 loc) · 4.18 KB
/
legacy-checkpoints.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: Create Legacy Ckpts
on:
workflow_dispatch:
inputs:
push_to_s3:
description: 'Push generated checkpoints to S3.'
default: false
required: false
type: boolean
pl_version:
description: 'PL version to generate a legacy checkpoint with. If not provided, uses PL from source.'
default: ''
required: false
type: string
create_pr:
description: 'Create a PR to enable testing PL with a new checkpoint.'
default: false
required: false
type: boolean
workflow_call:
inputs:
push_to_s3:
description: 'Push generated checkpoints to S3.'
default: false
required: false
type: boolean
pl_version:
description: 'PL version to generate a legacy checkpoint with. If not provided, uses PL from source.'
default: ''
required: false
type: string
create_pr:
description: 'Create a PR to enable testing PL with a new checkpoint.'
default: false
required: false
type: boolean
secrets:
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_KEY_ID:
required: true
defaults:
run:
shell: bash
jobs:
create-legacy-ckpts:
runs-on: ubuntu-20.04
outputs:
pl-version: ${{ steps.decide-version.outputs.pl-version }}
defaults:
run:
working-directory: tests/legacy
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
# Python version here needs to be supported by all PL versions listed in back-compatible-versions.txt.
python-version: 3.8
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }}
aws-region: us-east-1
- name: Install PL from source
working-directory: ./
env:
PACKAGE_NAME: pytorch
FREEZE_REQUIREMENTS: 1
run: |
pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html
pip list
if: inputs.pl_version == ''
- name: Install PL version
run: |
pip install "pytorch-lightning==${{ inputs.pl_version }}" \
-f https://download.pytorch.org/whl/cpu/torch_stable.html
pip list
if: inputs.pl_version != ''
- name: Pull legacy checkpoints
working-directory: ./
run: bash .actions/pull_legacy_checkpoints.sh
- name: Decide PL version to create a PR with
id: decide-version
run: |
python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT
- name: Generate checkpoints
run: |
bash generate_checkpoints.sh ${{ inputs.pl_version }}
- name: Upload checkpoints to GitHub Actions artifact
uses: actions/upload-artifact@v3
with:
name: checkpoints-${{ github.sha }}
path: tests/legacy/checkpoints/
- name: Upload checkpoints to S3 (dryrun)
run: |
aws s3 sync --dryrun checkpoints/ s3://pl-public-data/legacy/checkpoints/
zip -r checkpoints.zip checkpoints
aws s3 cp --dryrun checkpoints.zip s3://pl-public-data/legacy/ --acl public-read
- name: Upload checkpoints to S3
run: |
aws s3 sync checkpoints/ s3://pl-public-data/legacy/checkpoints/
zip -r checkpoints.zip checkpoints
aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read
if: inputs.push_to_s3
adding-ckpt-test:
runs-on: ubuntu-20.04
if: inputs.create_pr
needs: create-legacy-ckpts
steps:
- uses: actions/checkout@v3
with:
ref: master
- name: Append a new version to legacy checkpoint list
run: echo ${{ needs.create-legacy-ckpts.outputs.pl-version }} >> tests/legacy/back-compatible-versions.txt
- name: Create Pull Request
uses: peter-evans/create-pull-request@v4
with:
title: Adding test for legacy checkpiont created with ${{ needs.create-legacy-ckpts.outputs.pl-version }}
delete-branch: true
labels: |
tests
pl