/
app-cloud-e2e.yml
198 lines (179 loc) · 7.13 KB
/
app-cloud-e2e.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
trigger:
tags:
include:
- '*'
branches:
include:
- "master"
- "release/*"
- "refs/tags/*"
schedules:
- cron: "0 0 * * *" # At the end of every day
displayName: Daily midnight testing
branches:
include:
- "master"
pr:
branches:
include:
- "master"
- "release/*"
paths:
include:
- ".azure/app-cloud-e2e.yml"
- "requirements/app/**"
- "src/lightning_app/**"
- "tests/tests_app/**"
- "examples/app_*/**" # some tests_app tests call examples files
- "tests/tests_app_examples/**"
- "setup.py"
- ".actions/**"
exclude:
- "requirements/app/docs.txt"
- "*.md"
- "**/*.md"
# variables are automatically exported as environment variables so this will override pip's default cache dir
variables:
- name: pip_cache_dir
value: $(Pipeline.Workspace)/.pip
- name: local_id
value: $(Build.BuildId)
- name: video_artifact_dir
value: ./videos
jobs:
- job: App_cloud_e2e_testing
pool: azure-cpus
container:
image: mcr.microsoft.com/playwright/python:v1.27.1-focal
options: "--shm-size=4gb"
strategy:
matrix:
'App: v0_app':
name: "v0_app"
'App: boring_app':
name: "boring_app"
'App: template_streamlit_ui':
name: "template_streamlit_ui"
'App: template_react_ui':
name: "template_react_ui"
# 'App: template_jupyterlab': # TODO: clarify where these files lives
# name: "template_jupyterlab"
'App: idle_timeout':
name: "idle_timeout"
'App: collect_failures':
name: "collect_failures"
'App: custom_work_dependencies':
name: "custom_work_dependencies"
'App: installation_commands_app':
name: "installation_commands_app"
'App: drive':
name: "drive"
'App: payload':
name: "payload"
'App: commands_and_api':
name: "commands_and_api"
'App: quick_start':
name: "quick_start"
timeoutInMinutes: "20"
cancelTimeoutInMinutes: "2"
# values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
workspace:
clean: all
steps:
- script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
displayName: "Set id for this PR"
condition: eq(variables['Build.Reason'], 'PullRequest')
- bash: |
whoami
mkdir -p $(video_artifact_dir)
printf "local id: $(local_id)\n"
python --version
pip --version
displayName: 'Info'
# TODO: we are testing it as `lightning`, so add also version for `lightning_app`
- bash: git restore . && python -m pip install -e . -r requirements/app/test.txt -r requirements/app/ui.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
displayName: 'Install Lightning & dependencies'
- bash: |
python -m pip install playwright
python -m playwright install # --with-deps
displayName: 'Install Playwright system dependencies'
# The magic happens here it doesn't need to install the quick start dependencies.
# This test is very important to test the main user story of lightning app.
# It also e2e tests running on cloud without installing dependencies.
- bash: |
rm -rf examples/lightning-quick-start || true
git clone https://github.com/Lightning-AI/lightning-quick-start examples/lightning-quick-start
condition: eq(variables['name'], 'quick_start')
displayName: 'Clone Quick start Repo'
# - bash: |
# rm -rf examples/app_template_jupyterlab || true
# git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
# cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
# condition: eq(variables['name'], 'template_jupyterlab')
# displayName: 'Clone Template Jupyter Lab Repo'
- bash: |
rm -rf examples/app_template_react_ui || true
git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
condition: eq(variables['name'], 'template_react_ui')
displayName: 'Clone Template React UI Repo'
- bash: python -m pip install -q -r .actions/requirements.txt
displayName: 'Install assistant dependencies'
# Fix imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
# TODO: fixme when installing `lightning_app` as well
- bash: |
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
displayName: 'Adjust examples'
- bash: |
pip --version
pip list
displayName: 'List pip dependency'
- bash: |
mkdir -p ${VIDEO_LOCATION}
ls -l examples/${TEST_APP_NAME}
ls -l tests/tests_app_examples
python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=1200 --capture=no -v --color=yes
env:
HEADLESS: '1'
PACKAGE_LIGHTNING: '1'
CLOUD: '1'
VIDEO_LOCATION: $(video_artifact_dir)
PR_NUMBER: $(local_id)
TEST_APP_NAME: $(name)
HAR_LOCATION: './artifacts/hars'
SLOW_MO: '50'
#LAI_USER: $(LAI_USER) # for STAGING
#LAI_PASS: $(LAI_PASS) # for STAGING
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
LIGHTNING_DEBUG: '1'
displayName: 'Run the tests'
- task: PublishPipelineArtifact@1
condition: failed()
inputs:
path: $(video_artifact_dir)
artifactName: $(name)
displayName: 'Publish videos'
- bash: |
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
condition: always()
env:
#LAI_USER: $(LAI_USER) # for STAGING
#LAI_PASS: $(LAI_PASS) # for STAGING
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
PR_NUMBER: $(local_id)
TEST_APP_NAME: $(name)
# GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning
# GRID_USER_KEY: $(LIGHTNING_API_KEY) # TODO: clarify the meaning
# GRID_URL: $(LIGHTNING_CLOUD_URL)
# _GRID_USERNAME: $(LIGHTNING_USERNAME)
displayName: 'Clean Previous Apps'