/
config.yml
391 lines (330 loc) · 13.3 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
env: &env
environment:
GRUNTWORK_INSTALLER_VERSION: v0.0.36
MODULE_CI_VERSION: v0.46.0
MODULE_GCP_CI_VERSION: v0.1.1
TERRAFORM_VERSION: 1.1.4
PACKER_VERSION: 1.7.4
TERRAGRUNT_VERSION: v0.36.0
OPA_VERSION: v0.33.1
GO_VERSION: 1.17
GO111MODULE: auto
K8S_VERSION: v1.18.20 # Same as EKS
MINIKUBE_VERSION: v1.22.0
HELM_VERSION: v3.8.0
KUBECONFIG: /home/circleci/.kube/config
BIN_BUILD_PARALLELISM: 3
defaults: &defaults
machine:
enabled: true
image: ubuntu-2004:202111-02
<<: *env
setup_minikube: &setup_minikube
command: |
sudo apt update -y
sudo apt install -y conntrack
setup-minikube --k8s-version "$K8S_VERSION" --minikube-version "$MINIKUBE_VERSION"
install_helm: &install_helm
name: install helm
command: |
# install helm
curl -Lo helm.tar.gz https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz
tar -xvf helm.tar.gz
chmod +x linux-amd64/helm
sudo mv linux-amd64/helm /usr/local/bin/
install_gruntwork_utils: &install_gruntwork_utils
name: install gruntwork utils
command: |
curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}"
gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "${MODULE_CI_VERSION}"
gruntwork-install --module-name "kubernetes-circleci-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "${MODULE_CI_VERSION}"
gruntwork-install --module-name "gcp-helpers" --repo "https://github.com/gruntwork-io/terraform-google-ci" --tag "${MODULE_GCP_CI_VERSION}"
configure-environment-for-gruntwork-module \
--terraform-version ${TERRAFORM_VERSION} \
--terragrunt-version ${TERRAGRUNT_VERSION} \
--packer-version ${PACKER_VERSION} \
--go-version NONE
# Install OPA
echo "Installing OPA version ${OPA_VERSION}"
curl -sLO "https://github.com/open-policy-agent/opa/releases/download/${OPA_VERSION}/opa_linux_amd64_static"
chmod +x ./opa_linux_amd64_static
sudo mv ./opa_linux_amd64_static /usr/local/bin/opa
# Temporary fix for installing go - remove when we can update gruntwork-module-circleci-helpers to version with fix
echo "Installing Go version $version"
curl -O --silent --location --fail --show-error "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz"
sudo rm -rf /usr/local/go
sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
sudo ln -s /usr/local/go/bin/go /usr/bin/go
echo "The installed version of Go is now $(go version)"
install_docker_buildx: &install_docker_buildx
name: install docker buildx
command: |
curl -sLO https://github.com/docker/buildx/releases/download/v0.6.1/buildx-v0.6.1.linux-amd64
mkdir -p ~/.docker/cli-plugins
mv buildx-v0.6.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx
chmod a+x ~/.docker/cli-plugins/docker-buildx
# Verify buildx is available
docker buildx create --use
configure_environment_for_gcp: &configure_environment_for_gcp
name: configure environment for gcp
command: |
# install the Google Cloud SDK CLI
install-gcloud
# Make GCP Service Account credentials available as a file
echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
echo 'export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcloud-service-key.json' >> $BASH_ENV
# Tell gcloud to use the credentials and set defaults
echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
gcloud --quiet config set compute/zone ${GOOGLE_COMPUTE_ZONE}
version: 2
jobs:
setup:
<<: *env
docker:
- image: cimg/python:3.10.2
steps:
- checkout
- restore_cache:
keys:
- gomod-{{ checksum "go.sum" }}
# Install gruntwork utilities
- run:
<<: *install_gruntwork_utils
- save_cache:
key: gomod-{{ checksum "go.sum" }}
paths:
- $HOME/go/src/
# The weird way you have to set PATH in Circle 2.0
- run: |
echo 'export PATH=$HOME/.local/bin:$HOME/go/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
# Run pre-commit hooks and fail the build if any hook finds required changes.
- run:
name: run precommit
command: |
go install golang.org/x/tools/cmd/goimports@latest
# Install the latest minor version for v2
pip install pre-commit~=2.9
pre-commit install
pre-commit run --all-files
# Build any binaries that need to be built
# We always want to build the binaries to test that there are no compile failures. Also, we will use the
# terratest_log_parser to parse out the test output during a failure. Finally, on releases, we'll push these
# binaries to GitHub as release assets.
- run:
command: |
# For some reason, the circleci environment requires additional module dependencies that are not captured by
# our Linux or Mac OSX dev environments. We workaround this by running `go mod tidy` in the CircleCI
# environment so it pulls in what it needs.
go mod tidy
go install github.com/mitchellh/gox@latest
GO_ENABLED=0 build-go-binaries \
--parallel "$BIN_BUILD_PARALLELISM" \
--app-name terratest_log_parser \
--src-path ./cmd/terratest_log_parser \
--dest-path ./cmd/bin \
--ld-flags "-X main.VERSION=$CIRCLE_TAG -extldflags '-static'"
GO_ENABLED=0 build-go-binaries \
--parallel "$BIN_BUILD_PARALLELISM" \
--app-name pick-instance-type \
--src-path ./cmd/pick-instance-type \
--dest-path ./cmd/bin \
--ld-flags "-X main.VERSION=$CIRCLE_TAG -extldflags '-static'"
when: always
- persist_to_workspace:
root: /home/circleci
paths:
- project
test:
<<: *defaults
resource_class: large
steps:
- attach_workspace:
at: /home/circleci
- run:
<<: *install_gruntwork_utils
- run:
<<: *install_docker_buildx
# The weird way you have to set PATH in Circle 2.0
- run: |
echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
# Run the tests. Note that we set the "-p 1" flag to tell Go to run tests in each package sequentially. Without
# this, Go buffers all log output until all packages are done, which with slower running tests can cause CircleCI
# to kill the build after more than 10 minutes without log output.
# NOTE: because this doesn't build with the kubernetes tag, it will not run the kubernetes tests. See
# kubernetes_test build steps.
- run: mkdir -p /tmp/logs
# check we can compile the azure code, but don't actually run the tests
- run: run-go-tests --packages "-p 1 -tags=azure -run IDontExist ./modules/azure"
- run: run-go-tests --packages "-p 1 ./..." | tee /tmp/logs/test_output.log
- run:
command: |
./cmd/bin/terratest_log_parser_linux_amd64 --testlog /tmp/logs/test_output.log --outputdir /tmp/logs
when: always
# Store test result and log artifacts for browsing purposes
- store_artifacts:
path: /tmp/logs
- store_test_results:
path: /tmp/logs
# We run the GCP tests in a separate build step using the Docker executor for better isolation and resiliency. Using
# The Docker executor ensures GCP tests do not erroneously make metadata network calls within CircleCI's private
# environment. For more information see: https://github.com/gruntwork-io/terratest/pull/765.
gcp_test:
<<: *env
docker:
- image: cimg/base:2022.03
steps:
- attach_workspace:
at: /home/circleci
- run:
<<: *install_gruntwork_utils
# The weird way you have to set PATH in Circle 2.0
- run: |
echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
- run:
<<: *configure_environment_for_gcp
# Run the GCP tests. These tests are run because the gcp build tag is included, and we explicitly
# select the GCP tests
- run:
command: |
mkdir -p /tmp/logs
# Run the unit tests first, then the integration tests. They are separate because the integration tests
# require additional filtering.
run-go-tests --packages "-tags gcp ./modules/gcp" | tee /tmp/logs/test_output.log
run-go-tests --packages "-tags=gcp ./test/gcp" | tee /tmp/logs/test_output.log
- run:
command: |
./cmd/bin/terratest_log_parser_linux_amd64 --testlog /tmp/logs/test_output.log --outputdir /tmp/logs
when: always
# Store test result and log artifacts for browsing purposes
- store_artifacts:
path: /tmp/logs
- store_test_results:
path: /tmp/logs
kubernetes_test:
<<: *defaults
steps:
- attach_workspace:
at: /home/circleci
- run:
<<: *install_gruntwork_utils
# The weird way you have to set PATH in Circle 2.0
- run: |
echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
- run:
<<: *setup_minikube
# Run the Kubernetes tests. These tests are run because the kubernetes build tag is included, and we explicitly
# select the kubernetes tests
- run:
command: |
mkdir -p /tmp/logs
# Run the unit tests first, then the integration tests. They are separate because the integration tests
# require additional filtering.
run-go-tests --packages "-tags kubernetes ./modules/k8s" | tee /tmp/logs/test_output.log
run-go-tests --packages "-tags kubernetes -run TestKubernetes ./test" | tee -a /tmp/logs/test_output.log
- run:
command: |
./cmd/bin/terratest_log_parser_linux_amd64 --testlog /tmp/logs/test_output.log --outputdir /tmp/logs
when: always
# Store test result and log artifacts for browsing purposes
- store_artifacts:
path: /tmp/logs
- store_test_results:
path: /tmp/logs
helm_test:
<<: *defaults
steps:
- attach_workspace:
at: /home/circleci
- run:
<<: *install_gruntwork_utils
# The weird way you have to set PATH in Circle 2.0
- run: |
echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
- run:
<<: *setup_minikube
- run:
<<: *install_helm
# Run the Helm tests. These tests are run because the helm build tag is included, and we explicitly
# select the helm tests
- run:
command: |
mkdir -p /tmp/logs
# Run the unit tests first, then the integration tests. They are separate because the integration tests
# require additional filtering.
run-go-tests --packages "-tags helm ./modules/helm" | tee /tmp/logs/test_output.log
run-go-tests --packages "-tags helm -run TestHelm ./test" | tee -a /tmp/logs/test_output.log
- run:
command: |
./cmd/bin/terratest_log_parser_linux_amd64 --testlog /tmp/logs/test_output.log --outputdir /tmp/logs
when: always
# Store test result and log artifacts for browsing purposes
- store_artifacts:
path: /tmp/logs
- store_test_results:
path: /tmp/logs
deploy:
<<: *defaults
steps:
- checkout
- attach_workspace:
at: /home/circleci
- run: |
curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "$GRUNTWORK_INSTALLER_VERSION"
- run: |
gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "$MODULE_CI_VERSION"
- run: cd cmd/bin && sha256sum * > SHA256SUMS
- run: upload-github-release-assets cmd/bin/*
workflows:
version: 2
build-and-test:
jobs:
- setup:
context:
- Gruntwork Admin
filters:
tags:
only: /^v.*/
- kubernetes_test:
context:
- Gruntwork Admin
requires:
- setup
filters:
tags:
only: /^v.*/
- helm_test:
context:
- Gruntwork Admin
requires:
- setup
filters:
tags:
only: /^v.*/
- test:
context:
- Gruntwork Admin
- Slack Token For Test
requires:
- setup
filters:
tags:
only: /^v.*/
- gcp_test:
context:
- Gruntwork GCP
requires:
- setup
filters:
tags:
only: /^v.*/
- deploy:
context:
- Gruntwork Admin
requires:
- setup
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/