-
Notifications
You must be signed in to change notification settings - Fork 618
/
config.yml
1522 lines (1492 loc) · 53.7 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
version: 2.1
orbs:
win: circleci/windows@5.0.0
slack: circleci/slack@4.10.1
go: circleci/go@1.7.1
gke: circleci/gcp-gke@1.4.0
parameters:
manual:
type: boolean
default: false
manual_test:
type: boolean
default: false
manual_win:
type: boolean
default: false
manual_mac:
type: boolean
default: false
manual_test_image:
type: string
default: "python:3.7"
manual_test_toxenv:
type: string
default: "py37"
manual_win_toxenv:
type: string
default: "py37"
manual_mac_toxenv:
type: string
default: "py37"
manual_test_name:
type: string
default: "man-lin-py37"
manual_win_name:
type: string
default: "man-win-py37"
manual_mac_name:
type: string
default: "man-mac-py37"
manual_parallelism:
type: integer
default: 1
manual_xdist:
type: integer
default: 1
tox_version:
type: string
default: "3.24.0"
container_registry:
type: string
default: "gcr.io"
gcp_cluster_name:
type: string
default: "gke-yea"
manual_nightly:
type: boolean
default: false
manual_nightly_git_branch:
type: string
default: main
manual_nightly_slack_notify:
type: boolean
default: false
manual_nightly_execute_shard_standalone_cpu:
type: boolean
default: false
manual_nightly_execute_shard_standalone_gpu:
type: boolean
default: false
manual_nightly_execute_shard_standalone_tpu:
type: boolean
default: false
manual_nightly_execute_shard_standalone_local:
type: boolean
default: false
manual_nightly_execute_shard_kfp:
type: boolean
default: false
manual_nightly_execute_shard_standalone_gpu_win:
type: boolean
default: false
manual_nightly_execute_shard_imports:
type: boolean
default: false
wandb_server_tag:
type: string
default: "master"
commands:
save-test-results:
description: "Save test results"
steps:
- unless:
condition: << pipeline.parameters.manual >>
steps:
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
- store_artifacts:
path: mypy-results
- store_artifacts:
path: cover-results
setup_docker_buildx:
description: Enable remote docker, and install the expanded `buildx` command for the CLI. Only works on alpine-based images.
parameters:
docker_layer_caching:
type: boolean
default: false
steps:
- setup_remote_docker:
version: 20.10.12
docker_layer_caching: << parameters.docker_layer_caching >>
- run: apk add docker-cli-buildx --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
setup_gcloud:
parameters:
cluster:
description: "cluster name"
type: string
default: << pipeline.parameters.gcp_cluster_name >>
steps:
- run:
name: "Setup gcloud"
command: |
echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
gcloud --quiet config set project $GOOGLE_PROJECT_ID
gcloud --quiet config set compute/zone $GOOGLE_COMPUTE_ZONE
gcloud auth configure-docker --quiet << pipeline.parameters.container_registry >>
environment:
GKE_CLUSTER_NAME: <<parameters.cluster >>
create_gke_cluster:
parameters:
cluster:
description: "cluster name"
type: string
default: << pipeline.parameters.gcp_cluster_name >>
no-output-timeout:
description: >
Elapsed time that the cluster creation command can run on CircleCI without
output. The string is a decimal with unit suffix, such as “20m”, “1.25h”,
“5s”
type: string
default: 15m
num_nodes:
description: "Number of nodes to create"
type: integer
default: 1
machine_type:
description: "GKE cluster machine type"
type: string
default: "n1-standard-8"
disk_size:
description: "GKE cluster disk size"
type: integer
default: 100
disk_type:
description: "GKE cluster disk type"
type: string
default: "pd-ssd"
gpu_type:
description: "GKE cluster GPU type"
type: string
default: "nvidia-tesla-t4"
gpu_count:
description: "GKE cluster GPU count"
type: integer
default: 2
steps:
- setup_gcloud
- run:
name: "Check if cluster exists and delete if it does"
command: |
cluster_exists=$(gcloud container clusters list --filter="name=<< parameters.cluster >>" --format="value(name)" | wc -l | xargs)
gcloud container clusters list --filter="name=<< parameters.cluster >>" --format="value(name)"
if [ $? -eq 0 ] && [ $cluster_exists -eq 1 ]; then
gcloud container clusters delete << parameters.cluster >> --quiet
fi
exit 0
- run:
name: "Create GKE cluster"
command: |
gcloud container clusters create $GKE_CLUSTER_NAME \
--num-nodes=<< parameters.num_nodes >> --machine-type=<< parameters.machine_type >> \
--disk-size=<< parameters.disk_size >> --disk-type=<< parameters.disk_type >> \
--accelerator=type=<< parameters.gpu_type >>,count=<< parameters.gpu_count >>
environment:
GKE_CLUSTER_NAME: << parameters.cluster >>
no_output_timeout: << parameters.no-output-timeout>>
get_gke_cluster_credentials:
parameters:
cluster:
description: "cluster name"
type: string
default: << pipeline.parameters.gcp_cluster_name >>
steps:
- run:
name: "Get gke cluster credentials"
command: |
gcloud components install gke-gcloud-auth-plugin
gcloud container clusters get-credentials << parameters.cluster >>
environment:
GKE_CLUSTER_NAME: << parameters.cluster >>
jobs:
tox-base:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 7
toxenv:
type: string
shard:
type: string
default: "base"
notify_on_failure:
type: boolean
default: false
notify_on_failure_channel:
type: string
default: "$SLACK_SDK_NIGHTLY_CI_CHANNEL"
parallelism:
type: integer
default: 1
execute:
type: boolean
default: true
docker:
- image: "python:<<parameters.python_version_major>>.<<parameters.python_version_minor>>"
parallelism: << parameters.parallelism >>
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- checkout
- when:
condition: << parameters.execute >>
steps:
- run:
name: Install system deps
command: |
apt-get update && apt-get install -y libsndfile1 ffmpeg
- run:
name: Install python dependencies
command: |
pip install tox==<< pipeline.parameters.tox_version >>
pip install -U pip
no_output_timeout: 5m
- run:
name: Run tests
no_output_timeout: 10m
command: |
CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" tox -v -e << parameters.toxenv >>
- save-test-results
# conditionally post a notification to slack if the job failed
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: << parameters.notify_on_failure_channel >>
pytest:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 7
toxenv:
type: string
notify_on_failure:
type: boolean
default: false
xdist:
type: integer
default: 6
test_path:
type: string
default: tests/unit_tests
docker:
- image: "python:<<parameters.python_version_major>>.<<parameters.python_version_minor>>"
- image: gcr.io/wandb-production/local-testcontainer:<< pipeline.parameters.wandb_server_tag >>
auth:
username: _json_key
password: $GCP_SERVICE_ACCOUNT_JSON_DECODED
environment:
CI: 1
WANDB_ENABLE_TEST_CONTAINER: true
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- checkout
- run:
name: Install system deps
command: |
apt-get update && apt-get install -y libsndfile1 ffmpeg
- run:
name: Install python dependencies
command: |
pip install tox==<< pipeline.parameters.tox_version >>
pip install -U pip
while ! curl http://localhost:8080/healthz; do sleep 1; done
no_output_timeout: 5m
- run:
name: Run tests
no_output_timeout: 10m
command: |
CI_PYTEST_PARALLEL=<< parameters.xdist >> \
tox -v -e <<parameters.toxenv>> \
-- --wandb-server-tag << pipeline.parameters.wandb_server_tag >> << parameters.test_path >>
- save-test-results
# conditionally post a notification to slack if the job failed
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
pex:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 8
docker:
- image: "python:<<parameters.python_version_major>>.<<parameters.python_version_minor>>"
resource_class: small
working_directory: /mnt/ramdisk
steps:
- checkout
- run:
name: Install system deps
command: |
apt-get update && apt-get install -y libsndfile1 ffmpeg
- run:
name: Install pex
command: |
pip install -U pip
pip install pex
no_output_timeout: 5m
- run:
name: Run pex test
no_output_timeout: 5m
command: |
pex . -o wandb.pex
./wandb.pex -c "import wandb; wandb.init(mode='offline'); wandb.finish()"
- save-test-results
protobuf-compatability:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 8
docker:
- image: "python:<<parameters.python_version_major>>.<<parameters.python_version_minor>>"
resource_class: small
working_directory: /mnt/ramdisk
steps:
- checkout
- run:
name: Install system deps
command: |
apt-get update && apt-get install -y libsndfile1 ffmpeg
- run:
name: Check protobuf version compatibility
command: |
pip install -U pip
pip install .
python tools/check-protobuf-version-compatibility.py
no_output_timeout: 5m
- save-test-results
win:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 9
toxenv:
type: string
parallelism:
type: integer
default: 4
xdist:
type: integer
default: 3
machine_executor:
type: string
default: "default" # "default" or "server-2019-cuda"
executor_size:
type: string
default: "large" # could only be "medium" for "server-2019-cuda"
execute:
type: boolean
default: true
executor:
name: win/<< parameters.machine_executor >>
size: << parameters.executor_size >>
parallelism: << parameters.parallelism >>
steps:
- checkout
- when:
condition: << parameters.execute >>
steps:
- run:
name: Install python dependencies
shell: bash.exe
command: |
pip install tox==<< pipeline.parameters.tox_version >>
- run:
name: Temporary conda hack
shell: bash.exe
command: |
cp /c/tools/miniconda3/python* /c/tools/miniconda3/lib/venv/scripts/nt/
- when:
condition:
equal: [ "server-2019-cuda", << parameters.machine_executor >> ]
steps:
- run:
name: Update tox.ini on a GPU machine to install the proper pytorch version
shell: bash.exe
command: |
CUDA_VERSION=`ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA"`
CUDA_VERSION_NO_DOT=`echo "${CUDA_VERSION//./}"`
v=${CUDA_VERSION_NO_DOT:1}
sed -i -e "s/whl\/cpu/whl\/cu$v/g" tox.ini
- run:
name: Run tests
shell: bash.exe
command: |
echo $GCLOUD_SERVICE_KEY > key.json
gcloud auth activate-service-account --key-file=key.json
yes | gcloud auth configure-docker
DATE=$(date -u +%Y%m%d) CI_PYTEST_PARALLEL=<< parameters.xdist >> CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" tox -v -e << parameters.toxenv >>
no_output_timeout: 10m
- save-test-results
mac:
parameters:
python_version_major:
type: integer
default: 3
python_version_minor:
type: integer
default: 7
toxenv:
type: string
parallelism:
type: integer
default: 4
xdist:
type: integer
default: 3
macos:
xcode: 13.4.1
resource_class: large
parallelism: << parameters.parallelism >>
steps:
- checkout
- run:
name: Install python dependencies
command: |
pip3 install tox==<< pipeline.parameters.tox_version >>
- run:
name: Run tests
# Tests failed with Too many open files, so added ulimit
command: |
ulimit -n 4096
CI_PYTEST_PARALLEL=<< parameters.xdist >> CI_PYTEST_SPLIT_ARGS="--splits $CIRCLE_NODE_TOTAL --group $(( $CIRCLE_NODE_INDEX + 1 ))" python3 -m tox -v -e << parameters.toxenv >>
no_output_timeout: 10m
- save-test-results
launch:
parameters:
toxenv:
type: string
machine:
image: ubuntu-2004:202104-01
docker_layer_caching: true
resource_class: large
steps:
- attach_workspace:
at: .
- checkout
- run:
name: Install python dependencies, build r2d
command: |
pip3 install tox==<< pipeline.parameters.tox_version >>
pip3 install chardet
pip3 install iso8601
- run:
name: pull base docker images
command: |
docker pull nvidia/cuda:10.0-runtime
docker pull python:3.6-buster
- run:
name: Run tests
command: |
python3 -m tox -vv -e << parameters.toxenv >>
no_output_timeout: 10m
- save-test-results
kfp:
parameters:
toxenv:
type: string
execute:
type: boolean
default: true
notify_on_failure:
type: boolean
default: true
notify_on_success:
type: boolean
default: false
machine:
image: ubuntu-2004:2022.04.1
docker_layer_caching: true
resource_class: large
steps:
- attach_workspace:
at: .
- checkout
- when:
condition: << parameters.execute >>
steps:
- run:
name: Install system deps
command: sudo apt update && sudo apt-get install -y libsndfile1 ffmpeg
- run:
name: Use py37
command: |
pyenv install -s 3.7.13
pyenv versions
pyenv global 3.7.13
- run:
name: Install python dependencies
command: |
pip install --upgrade pip
pip install tox==<< pipeline.parameters.tox_version >>
- run:
name: Install kubectl
command: |
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
- run:
name: Install kind
command: |
curl -Lo ./kind/kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 --create-dirs
chmod +x ./kind/kind
export PATH="./kind:$PATH"
kind create cluster
- run:
name: Install KFP
command: |
export PIPELINE_VERSION=1.7.1
kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=$PIPELINE_VERSION"
sleep 10s
kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/env/platform-agnostic-pns?ref=$PIPELINE_VERSION"
sleep 10s
kubectl wait --for condition=ready --timeout=300s pods -n kubeflow --all
kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80 &
- run:
name: Run tests
command: |
tox -v -e << parameters.toxenv >>
no_output_timeout: 25m
# conditionally post a notification to slack if the job failed/succeeded
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
- when:
condition: << parameters.notify_on_success >>
steps:
- slack/notify:
event: pass
template: basic_success_1
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
- save-test-results
slack_notify:
parameters:
message:
type: string
default: ":runner:"
execute:
type: boolean
default: true
docker:
- image: "cimg/base:stable"
steps:
- when:
condition: << parameters.execute >>
steps:
- slack/notify:
custom: |
{
"blocks": [
{
"type": "section",
"fields": [
{
"type": "plain_text",
"text": "<< parameters.message >>",
"emoji": true
}
]
}
]
}
event: always
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
# this is to make sure `steps` is not empty
- run:
name: Print message to stdout
command: echo << parameters.message >>
# Build the docker image for a yea shard of the nightly workflow.
sdk_docker:
parameters:
description:
type: string
path:
type: string
image_name:
type: string
python_version:
type: string
default: "3.8"
git_branch:
type: string
default: "main"
execute:
type: boolean
default: true
notify_on_failure:
type: boolean
default: false
docker:
- image: "google/cloud-sdk:alpine"
steps:
- checkout
- when:
condition: << parameters.execute >>
steps:
- gke/install
- run:
name: "Update gcloud components"
command: gcloud --quiet components update
- setup_docker_buildx:
docker_layer_caching: false
- run:
name: "Build << parameters.description >>"
command: |
cd << parameters.path >>
docker build \
-t << parameters.image_name >> \
--build-arg PYTHON_VERSION=<< parameters.python_version >> \
--build-arg GIT_BRANCH=<< parameters.git_branch >> \
--build-arg UTC_DATE=$(date -u +%Y%m%d) \
.
- setup_gcloud
- run:
name: "Push << parameters.description >> to container registry"
command: |
docker push << parameters.image_name >>
# todo: clean up old images in gcr.io
# conditionally post a notification to slack if the job failed
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
# Nightly workflow for the SDK.
# - spin_up_cluster job creates a GKE cluster
# - individual test jobs use the cluster created by the spin_up_cluster job
# - each requires spin_up_cluster to be run first
# - each runs in parallel
# - installs gcloud and gke and spins up a pod, deploys the test,
# then polls the pod for results, and finally deletes the pod
# - shut_down_cluster job that deletes the cluster created by the spin_up_cluster job
# - requires spin_up_cluster to be run first
# -
# manage cluster: spin it up and shut it down
spin_up_cluster:
parameters:
cluster:
type: string
default: << pipeline.parameters.gcp_cluster_name >>
notify_on_failure:
type: boolean
default: false
docker:
- image: "cimg/base:stable"
steps:
- checkout
- go/install
- gke/install
- run:
name: "Update gcloud components"
command: gcloud --quiet components update
- create_gke_cluster:
cluster: << parameters.cluster >>
- run:
name: "Install GPU drivers"
command: kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded-latest.yaml
# conditionally post a notification to slack if the job failed
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
shut_down_cluster:
parameters:
cluster:
type: string
default: << pipeline.parameters.gcp_cluster_name >>
sleep_time: # time in seconds to wait before re-checking pod status
type: integer
default: 30
notify_on_failure:
type: boolean
default: false
docker:
- image: "cimg/base:stable"
steps:
- checkout
- go/install
- gke/install
- run:
name: "Update gcloud components"
command: gcloud --quiet components update
- setup_gcloud
- get_gke_cluster_credentials:
cluster: << parameters.cluster >>
- run:
# todo: think about how to check statuses of pods before deleting cluster
name: "Wait for pods to finish"
command: |
echo "Waiting for pods to finish"
sleep 300
count_all_pods_terminated=0
while true; do
kubectl get pods
terminated_pods=`kubectl get pods --field-selector=status.phase!=Succeeded,status.phase!=Failed 2>&1`
all_pods_terminated=`echo $(echo $terminated_pods | grep -c "No resources found")`
if [ "$all_pods_terminated" -eq "1" ]; then
count_all_pods_terminated=$((count_all_pods_terminated + 1))
echo "All pods have terminated ($count_all_pods_terminated)"
# make sure we have seen all pods terminated twice to give pod attach a chance to run
if [ $count_all_pods_terminated -ge 2 ]; then break; fi
fi
sleep << parameters.sleep_time >>
done
no_output_timeout: "25m"
- run:
name: "Delete cluster"
command: gcloud container clusters delete << parameters.cluster >>
when: always
no_output_timeout: "10m"
# conditionally post a notification to slack if the job failed
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
cloud_test:
parameters:
cluster:
type: string
default: << pipeline.parameters.gcp_cluster_name >>
image_name:
type: string
path:
type: string
pod_config_name:
type: string
default: pod.yaml
pod_name: # same as in <path>/pod.yaml
type: string
sleep_time: # time in seconds to wait before re-checking pod status
type: integer
default: 5
notify_on_failure:
type: boolean
default: true
notify_on_success:
type: boolean
default: false
execute:
type: boolean
default: true
docker:
- image: "cimg/base:stable"
steps:
- checkout
- when:
condition: << parameters.execute >>
steps:
- go/install
- gke/install
- run:
name: "Update gcloud components"
command: gcloud --quiet components update
- setup_gcloud
- get_gke_cluster_credentials:
cluster: << parameters.cluster >>
- run:
name: "Propagate the WANDB_API_KEY environment variable to pod.yaml"
command: |
sed -i -e 's/WANDB_API_KEY_PLACEHOLDER/'"$WANDB_API_KEY"'/g' << parameters.path >>/<< parameters.pod_config_name >>
- run:
name: "Spin up pod"
command: |
kubectl apply -f << parameters.path >>/<< parameters.pod_config_name >>
- run:
name: "Wait for pod to be up and running"
command: |
while true; do
kubectl get pods
pod_state=$(kubectl get pods | grep << parameters.pod_name >>)
if [ "$(echo "$pod_state" | grep -c 'Running')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> is ready"
exit 0
elif [ "$(echo "$pod_state" | grep -c 'Completed')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> has completed"
exit 0
elif [ "$(echo "$pod_state" | grep -c 'Error')" -eq "1" ] || [ "$(echo "$pod_state" | grep -c 'CrashLoopBackOff')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> is in an error state"
kubectl logs << parameters.pod_name >>
exit 1
fi
sleep << parameters.sleep_time >>
done
no_output_timeout: "5m"
- run:
name: "Wait for tests to finish"
command: |
sleep 30
while true; do
kubectl get pods
pod_state=$(kubectl get pods | grep << parameters.pod_name >>)
if [ "$(echo "$pod_state" | grep -c 'Running')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> is running"
elif [ "$(echo "$pod_state" | grep -c 'Completed')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> has completed"
break
elif [ "$(echo "$pod_state" | grep -c 'Error')" -eq "1" ] || [ "$(kubectl get pods | grep -c 'CrashLoopBackOff')" -eq "1" ]; then
echo "Pod for << parameters.image_name >> is in an error state"
kubectl logs << parameters.pod_name >>
exit 1
fi
echo "Sleeping for << parameters.sleep_time >> seconds"
echo
sleep << parameters.sleep_time >>
done
no_output_timeout: "20m"
- run:
name: "Grab results"
command: |
# grab result data
echo "Launch attach pod to get results..."
kubectl create -f << parameters.path >>/attach.yaml
while true; do
kubectl get pods
pod_state=$(kubectl get pods | grep << parameters.pod_name >>-attach-pod)
if [ "$(echo "$pod_state" | grep -c 'Running')" -eq "1" ]; then
echo "Attach Pod for << parameters.image_name >> is running"
break
elif [ "$(echo "$pod_state" | grep -c 'Completed')" -eq "1" ]; then
echo "Attach Pod for << parameters.image_name >> has completed"
break
elif [ "$(echo "$pod_state" | grep -c 'Error')" -eq "1" ] || [ "$(kubectl get pods | grep -c 'CrashLoopBackOff')" -eq "1" ]; then
echo "Attach Pod for << parameters.image_name >> is in an error state"
kubectl logs << parameters.pod_name >>-attach-pod
echo "About to exit"
exit 1
fi
echo "Sleeping for << parameters.sleep_time >> seconds"
echo
sleep << parameters.sleep_time >>
done
echo "Copy results from pod..."
mkdir -p results/<< parameters.pod_name >>
kubectl cp << parameters.pod_name >>-attach-pod:/wandb-store/test-results results/<< parameters.pod_name >>
echo "Delete attach pod..."
kubectl delete pod << parameters.pod_name >>-attach-pod
no_output_timeout: "10m"
- run:
# todo: make getting the exit_code more robust: add check for "commands succeeded"
name: "Get pod logs and parse exit code"
command: |
kubectl logs << parameters.pod_name >>
logs=`kubectl logs << parameters.pod_name >>`
exit_code=`echo $(echo $logs | grep -c "commands failed")`
echo "Pod for << parameters.image_name >> exited with code ${exit_code}"
exit ${exit_code}
no_output_timeout: "20m"
- store_test_results:
path: results/<< parameters.pod_name >>/test-results
- store_artifacts:
path: results/<< parameters.pod_name >>/test-results
destination: test-results
- run:
name: "Delete the pod"
when: always
command: |
kubectl get pods
kubectl delete -f << parameters.path >>/<< parameters.pod_config_name >> || echo "Problem deleting pod"
kubectl delete -f << parameters.path >>/attach.pod || echo "Problem deleting attach pod"
# conditionally post a notification to slack if the job failed/succeeded
- when:
condition: << parameters.notify_on_failure >>
steps:
- slack/notify:
event: fail
template: basic_fail_1
mentions: "@channel"
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
- when:
condition: << parameters.notify_on_success >>
steps:
- slack/notify:
event: pass
template: basic_success_1
# taken from slack-secrets context
channel: $SLACK_SDK_NIGHTLY_CI_CHANNEL
pip_install_wandb:
parameters:
python_version:
type: string
notify_on_failure:
type: boolean
default: true
notify_on_success:
type: boolean
default: false
docker:
- image: "python:<< parameters.python_version >>"
steps:
- run:
name: "Install stable version of wandb with stable versions of dependencies"
command: |
pip install wandb
python -c "import wandb"
pip freeze > deps_wandb.txt
pip uninstall -y -r deps_wandb.txt
when: always