forked from openshift/assisted-test-infra
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
334 lines (242 loc) · 13.6 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
#############
# Variables #
#############
SHELL=/bin/sh
CONTAINER_COMMAND = $(shell if [ -x "$(shell command -v docker)" ];then echo "docker" ; else echo "podman";fi)
PULL_PARAM=$(shell if [ "${CONTAINER_COMMAND}" = "podman" ];then echo "--pull-always" ; else echo "--pull";fi)
ROOT_DIR = $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
REPORTS = $(ROOT_DIR)/reports
SKIPPER_PARAMS ?= -i
# assisted-service
SERVICE_BRANCH := $(or $(SERVICE_BRANCH), "master")
SERVICE_REPO := $(or $(SERVICE_REPO), "https://github.com/openshift/assisted-service")
SERVICE := $(or $(SERVICE), quay.io/ocpmetal/assisted-service:latest)
SERVICE_NAME := $(or $(SERVICE_NAME),assisted-service)
# ui service
UI_SERVICE_NAME := $(or $(UI_SERVICE_NAME),ocp-metal-ui)
# nodes params
ISO := $(or $(ISO), "") # ISO should point to a file that has the '.iso' extension. Otherwise deploy will fail!
NUM_MASTERS := $(or $(NUM_MASTERS),3)
WORKER_MEMORY ?= 8892
MASTER_MEMORY ?= 16984
NUM_WORKERS := $(or $(NUM_WORKERS),0)
NUM_DAY2_WORKERS := $(or $(NUM_DAY2_WORKERS),1)
STORAGE_POOL_PATH := $(or $(STORAGE_POOL_PATH), $(PWD)/storage_pool)
CLUSTER_ID := $(or $(CLUSTER_ID), "")
CLUSTER_NAME := $(or $(CLUSTER_NAME),test-infra-cluster)
OPENSHIFT_VERSION := $(or $(OPENSHIFT_VERSION), 4.6)
REMOTE_SERVICE_URL := $(or $(REMOTE_SERVICE_URL), "")
WORKER_DISK ?= 21474836480
MASTER_DISK ?= 128849018880
# network params
NAMESPACE := $(or $(NAMESPACE),assisted-installer)
BASE_DNS_DOMAINS := $(or $(BASE_DNS_DOMAINS), "")
BASE_DOMAIN := $(or $(BASE_DOMAIN),redhat.com)
NETWORK_CIDR := $(or $(NETWORK_CIDR), "")
NETWORK_MTU := $(or $(NETWORK_MTU), 1500)
HTTP_PROXY_URL := $(or $(HTTP_PROXY_URL), "")
HTTPS_PROXY_URL := $(or $(HTTPS_PROXY_URL), "")
NO_PROXY_VALUES := $(or $(NO_PROXY_VALUES), "")
VIP_DHCP_ALLOCATION := $(or $(VIP_DHCP_ALLOCATION),yes)
#day2 params
API_VIP_IP := $(or $(API_VIP_IP),"")
API_VIP_DNSNAME := $(or $(API_VIP_DNSNAME),"")
# secrets
SSH_PUB_KEY := $(or $(SSH_PUB_KEY),$(shell cat ssh_key/key.pub))
PULL_SECRET := $(or $(PULL_SECRET), $(shell if ! [ -z "${PULL_SECRET_FILE}" ];then cat ${PULL_SECRET_FILE};fi))
ROUTE53_SECRET := $(or $(ROUTE53_SECRET), "")
# deploy
IMAGE_TAG := latest
DEPLOY_TAG := $(or $(DEPLOY_TAG), "")
DEPLOY_MANIFEST_PATH := $(or $(DEPLOY_MANIFEST_PATH), "")
DEPLOY_MANIFEST_TAG := $(or $(DEPLOY_MANIFEST_TAG), "")
IMAGE_NAME=test-infra
IMAGE_REG_NAME=quay.io/itsoiref/$(IMAGE_NAME)
# oc deploy
KUBECONFIG := $(or $(KUBECONFIG),${HOME}/.kube/config)
ifneq ($(or $(OC_MODE),),)
OC_FLAG := --oc-mode
OC_TOKEN := $(or $(OC_TOKEN),"")
OC_SERVER := $(or $(OC_SERVER),https://api.ocp.prod.psi.redhat.com:6443)
OC_SCHEME := $(or $(OC_SCHEME),http)
OC_PARAMS = $(OC_FLAG) -oct $(OC_TOKEN) -ocs $(OC_SERVER) --oc-scheme $(OC_SCHEME)
endif
ifdef KEEP_ISO
KEEP_ISO_FLAG = --keep-iso
endif
SSO_URL := $(or $(SSO_URL), https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token)
OCM_BASE_URL := $(or $(OCM_BASE_URL), https://api-integration.6943.hive-integration.openshiftapps.com)
# minikube profile is used to manage multiple minikube instances
PROFILE := $(or $(PROFILE),minikube)
DEPLOY_TARGET := $(or $(DEPLOY_TARGET),minikube)
OCP_KUBECONFIG := $(or $(OCP_KUBECONFIG),build/kubeconfig)
.EXPORT_ALL_VARIABLES:
.PHONY: image_build run destroy start_minikube delete_minikube run destroy install_minikube deploy_assisted_service create_environment delete_all_virsh_resources _download_iso _deploy_assisted_service _deploy_nodes _destroy_terraform
###########
# General #
###########
all: create_full_environment run_full_flow_with_install
destroy: destroy_nodes delete_minikube_profile kill_port_forwardings delete_podman_localhost
###############
# Environment #
###############
create_full_environment:
./create_full_environment.sh
create_environment: image_build bring_assisted_service start_minikube
image_build:
sed 's/^FROM .*assisted-service.*:latest/FROM $(subst /,\/,${SERVICE})/' Dockerfile.test-infra | \
$(CONTAINER_COMMAND) build --network=host ${PULL_PARAM} -t $(IMAGE_NAME):$(IMAGE_TAG) -f- .
clean:
-rm -rf build assisted-service test_infra.log
-find -name '*.pyc' -delete
-find -name '*pycache*' -delete
############
# Minikube #
############
install_minikube:
scripts/install_minikube.sh
start_minikube:
scripts/run_minikube.sh
eval $(minikube docker-env)
delete_minikube:
skipper run python3 scripts/indexer.py --action del --namespace all $(OC_FLAG)
minikube delete --all
skipper run discovery-infra/virsh_cleanup.py -m
delete_minikube_profile:
skipper run python3 scripts/indexer.py --action del --namespace $(NAMESPACE) $(OC_FLAG)
minikube delete -p $(PROFILE)
####################
# Podman localhost #
####################
delete_podman_localhost:
make -C assisted-service/ clean-onprem
#############
# Terraform #
#############
copy_terraform_files:
mkdir -p build/terraform/$(CLUSTER_NAME)__$(NAMESPACE)
FILE=build/terraform/$(CLUSTER_NAME)__$(NAMESPACE)/terraform.tfvars.json
cp -r terraform_files/* build/terraform/$(CLUSTER_NAME)__$(NAMESPACE);\
run_terraform: copy_terraform_files
skipper make $(SKIPPER_PARAMS) _run_terraform
_run_terraform:
cd build/terraform/$(CLUSTER_NAME)__$(NAMESPACE) && \
terraform init -plugin-dir=/root/.terraform.d/plugins/ && \
terraform apply -auto-approve -input=false -state=terraform.tfstate -state-out=terraform.tfstate -var-file=terraform.tfvars.json
_apply_terraform:
cd build/terraform/$(CLUSTER_NAME)__$(NAMESPACE) && \
terraform apply -auto-approve -input=false -state=terraform.tfstate -state-out=terraform.tfstate -var-file=terraform.tfvars.json
destroy_terraform:
skipper make $(SKIPPER_PARAMS) _destroy_terraform
_destroy_terraform:
cd build/terraform/ && terraform destroy -auto-approve -input=false -state=terraform.tfstate -state-out=terraform.tfstate -var-file=terraform.tfvars.json || echo "Failed cleanup terraform"
discovery-infra/virsh_cleanup.py -f test-infra
#######
# Run #
#######
validate_namespace:
scripts/utils.sh validate_namespace $(NAMESPACE)
run: validate_namespace deploy_assisted_service deploy_ui
run_full_flow: run deploy_nodes set_dns
redeploy_all: destroy run_full_flow
run_full_flow_with_install: run deploy_nodes_with_install set_dns
redeploy_all_with_install: destroy run_full_flow_with_install
set_dns:
scripts/assisted_deployment.sh set_dns $(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG))
deploy_ui: start_minikube
DEPLOY_TAG=$(DEPLOY_TAG) NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) DEPLOY_MANIFEST_PATH=$(DEPLOY_MANIFEST_PATH) DEPLOY_MANIFEST_TAG=$(DEPLOY_MANIFEST_TAG) scripts/deploy_ui.sh
test_ui: deploy_ui
DEPLOY_TAG=$(DEPLOY_TAG) DEPLOY_MANIFEST_PATH=$(DEPLOY_MANIFEST_PATH) DEPLOY_MANIFEST_TAG=$(DEPLOY_MANIFEST_TAG) PULL_SECRET=${PULL_SECRET} scripts/test_ui.sh
kill_port_forwardings:
scripts/utils.sh kill_port_forwardings '$(NAMESPACE)'
kill_all_port_forwardings:
scripts/utils.sh kill_port_forwardings '$(SERVICE_NAME) $(UI_SERVICE_NAME)'
deploy_on_ocp_cluster:
# service
DEPLOY_TARGET=ocp NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE)) \
DEPLOY_TAG=$(DEPLOY_TAG) DEPLOY_MANIFEST_TAG=$(DEPLOY_MANIFEST_TAG) OCP_KUBECONFIG=$(OCP_KUBECONFIG) \
CLUSTER_NAME=$(CLUSTER_NAME) CLUSTER_ID=$(CLUSTER_ID) PROFILE=$(PROFILE) \
scripts/deploy_assisted_service.sh
# UI
DEPLOY_TARGET=ocp NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE)) \
DEPLOY_TAG=$(DEPLOY_TAG) DEPLOY_MANIFEST_TAG=$(DEPLOY_MANIFEST_TAG) OCP_KUBECONFIG=$(OCP_KUBECONFIG) \
CLUSTER_NAME=$(CLUSTER_NAME) CLUSTER_ID=$(CLUSTER_ID) PROFILE=$(PROFILE) \
scripts/deploy_ui.sh
config_etc_hosts_for_ocp_cluster:
discovery-infra/ocp.py --config-etc-hosts -cn $(CLUSTER_NAME) -id $(CLUSTER_ID) -ns $(NAMESPACE) --service-name $(SERVICE_NAME) --profile $(PROFILE) --deploy-target ocp $(ADDITIONAL_PARAMS)
###########
# Cluster #
###########
_install_cluster:
discovery-infra/install_cluster.py -id $(CLUSTER_ID) -ps '$(PULL_SECRET)' --service-name $(SERVICE_NAME) $(OC_PARAMS) -ns $(NAMESPACE) -cn $(CLUSTER_NAME) --profile $(PROFILE)
install_cluster:
skipper make $(SKIPPER_PARAMS) _install_cluster NAMESPACE=$(NAMESPACE)
#########
# Nodes #
#########
_deploy_nodes:
discovery-infra/start_discovery.py -i $(ISO) -n $(NUM_MASTERS) -p $(STORAGE_POOL_PATH) -k '$(SSH_PUB_KEY)' -md $(MASTER_DISK) -wd $(WORKER_DISK) -mm $(MASTER_MEMORY) -wm $(WORKER_MEMORY) -nw $(NUM_WORKERS) -ps '$(PULL_SECRET)' -bd $(BASE_DOMAIN) -cN $(CLUSTER_NAME) -vN $(NETWORK_CIDR) -nM $(NETWORK_MTU) -ov $(OPENSHIFT_VERSION) -iU $(REMOTE_SERVICE_URL) -id $(CLUSTER_ID) -mD $(BASE_DNS_DOMAINS) -ns $(NAMESPACE) -pX $(HTTP_PROXY_URL) -sX $(HTTPS_PROXY_URL) -nX $(NO_PROXY_VALUES) --service-name $(SERVICE_NAME) --vip-dhcp-allocation $(VIP_DHCP_ALLOCATION) --profile $(PROFILE) --ns-index $(NAMESPACE_INDEX) --deploy-target $(DEPLOY_TARGET) $(DAY1_PARAMS) $(OC_PARAMS) $(KEEP_ISO_FLAG) $(ADDITIONAL_PARAMS) $(DAY2_PARAMS) -ndw $(NUM_DAY2_WORKERS)
deploy_nodes_with_install:
skipper make $(SKIPPER_PARAMS) _deploy_nodes NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE) ADDITIONAL_PARAMS=-in $(SKIPPER_PARAMS) DAY1_PARAMS=--day1-cluster
deploy_nodes:
skipper make $(SKIPPER_PARAMS) _deploy_nodes NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE) DAY1_PARAMS=--day1-cluster
_deploy_day2_nodes:
discovery-infra/start_discovery.py -i $(ISO) -k '$(SSH_PUB_KEY)' -nw $(NUM_WORKERS) -ps '$(PULL_SECRET)' -bd $(BASE_DOMAIN) -cN $(CLUSTER_NAME) -ov $(OPENSHIFT_VERSION) -id $(CLUSTER_ID) -ns $(NAMESPACE) -pX $(HTTP_PROXY_URL) -sX $(HTTPS_PROXY_URL) -nX $(NO_PROXY_VALUES) --service-name $(SERVICE_NAME) --profile $(PROFILE) --ns-index $(NAMESPACE_INDEX) --day2-cluster yes -avi $(API_VIP_IP) -avd $(API_VIP_DNSNAME) $(OC_PARAMS) $(KEEP_ISO_FLAG) $(ADDITIONAL_PARAMS)
deploy_day2_nodes:
skipper make $(SKIPPER_PARAMS) _deploy_nodes NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE) $(SKIPPER_PARAMS) DAY2_PARAMS=--day2-cluster
deploy_day2_nodes_with_install:
skipper make $(SKIPPER_PARAMS) _deploy_nodes NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE) $(SKIPPER_PARAMS) ADDITIONAL_PARAMS=-in DAY2_PARAMS=--day2-cluster
install_day1_and_day2:
skipper make $(SKIPPER_PARAMS) _deploy_nodes NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE) $(SKIPPER_PARAMS) ADDITIONAL_PARAMS=-in DAY2_PARAMS=--day2-cluster DAY1_PARAMS=--day1-cluster
destroy_nodes:
skipper run $(SKIPPER_PARAMS) 'discovery-infra/delete_nodes.py -iU $(REMOTE_SERVICE_URL) -id $(CLUSTER_ID) -ns $(NAMESPACE) --service-name $(SERVICE_NAME) --profile $(PROFILE) -cn $(CLUSTER_NAME) $(OC_PARAMS)'
rm -rf build/terraform/$(CLUSTER_NAME)__$(NAMESPACE)
destroy_all_nodes_from_namespaces:
skipper run $(SKIPPER_PARAMS) 'discovery-infra/delete_nodes.py -iU $(REMOTE_SERVICE_URL) -id $(CLUSTER_ID) -cn $(CLUSTER_NAME) --service-name $(SERVICE_NAME) $(OC_PARAMS) -ns all'
destroy_all_nodes:
skipper run $(SKIPPER_PARAMS) 'discovery-infra/delete_nodes.py --delete-all'
redeploy_nodes: destroy_nodes deploy_nodes
redeploy_nodes_with_install: destroy_nodes deploy_nodes_with_install
#############
# Inventory #
#############
deploy_assisted_service: start_minikube bring_assisted_service
mkdir -p assisted-service/build
DEPLOY_TAG=$(DEPLOY_TAG) NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) DEPLOY_MANIFEST_PATH=$(DEPLOY_MANIFEST_PATH) DEPLOY_MANIFEST_TAG=$(DEPLOY_MANIFEST_TAG) scripts/deploy_assisted_service.sh
bring_assisted_service:
@if cd assisted-service >/dev/null 2>&1; then git fetch --all && git reset --hard origin/$(SERVICE_BRANCH); else git clone --branch $(SERVICE_BRANCH) $(SERVICE_REPO);fi
deploy_monitoring: bring_assisted_service
make -C assisted-service/ deploy-monitoring NAMESPACE=$(NAMESPACE) PROFILE=$(PROFILE)
delete_all_virsh_resources: destroy_all_nodes delete_minikube kill_all_port_forwardings
skipper run $(SKIPPER_PARAMS) 'discovery-infra/delete_nodes.py -ns $(NAMESPACE) -a'
_download_logs:
discovery-infra/download_logs.py $(REMOTE_SERVICE_URL) $(LOGS_DEST) --cluster-id $(CLUSTER_ID)
download_logs:
skipper make $(SKIPPER_PARAMS) _download_logs
_download_all_logs:
discovery-infra/download_logs.py $(REMOTE_SERVICE_URL) $(LOGS_DEST) --download-all
download_all_logs:
skipper make $(SKIPPER_PARAMS) _download_all_logs
#######
# ISO #
#######
_download_iso:
discovery-infra/start_discovery.py -k '$(SSH_PUB_KEY)' -ps '$(PULL_SECRET)' -bd $(BASE_DOMAIN) -cN $(CLUSTER_NAME) -ov $(OPENSHIFT_VERSION) -pX $(HTTP_PROXY_URL) -sX $(HTTPS_PROXY_URL) -nX $(NO_PROXY_VALUES) -iU $(REMOTE_SERVICE_URL) -id $(CLUSTER_ID) -mD $(BASE_DNS_DOMAINS) -ns $(NAMESPACE) --service-name $(SERVICE_NAME) --profile $(PROFILE) --ns-index $(NAMESPACE_INDEX) $(OC_PARAMS) -iO
download_iso:
skipper make $(SKIPPER_PARAMS) _download_iso NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) NAMESPACE=$(NAMESPACE)
download_iso_for_remote_use: deploy_assisted_service
skipper make $(SKIPPER_PARAMS) _download_iso NAMESPACE_INDEX=$(shell bash scripts/utils.sh get_namespace_index $(NAMESPACE) $(OC_FLAG)) $(OC_FLAG)) NAMESPACE=$(NAMESPACE)
########
# Test #
########
$(REPORTS):
-mkdir -p $(REPORTS)
lint:
mkdir -p build
skipper make _lint
_lint:
pre-commit run --all-files
test:
skipper make $(SKIPPER_PARAMS) _test
_test: $(REPORTS)
python3 -m pytest $(or ${TEST},${TEST},discovery-infra/tests) --verbose -s --junit-xml=$(REPORTS)/unittest.xml