-
Notifications
You must be signed in to change notification settings - Fork 148
/
Makefile
247 lines (215 loc) · 9.88 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
# parameters
TEST_KUBERNETES_TARGET ?= current
TEST_SCHEDULER_MANIFEST ?= daemonset
STORAGE_CAPACITY ?= false
## Dependency versions
MINIKUBE_VERSION := v1.23.2
CERT_MANAGER_VERSION := v1.3.1
BINDIR := $(shell pwd)/bin
SUDO := sudo
KIND_CLUSTER_NAME := topolvm-e2e
KIND := ../bin/kind
KUBECTL := $(BINDIR)/kubectl
HELM := ../bin/helm
GINKGO := ../bin/ginkgo
MINIKUBE_HOME = $(BINDIR)
export MINIKUBE_HOME
READ_WRITE_ONCE_POD=false
ifeq ($(TEST_KUBERNETES_TARGET),current)
KUBERNETES_VERSION := 1.22.2
READ_WRITE_ONCE_POD=true
else ifeq ($(TEST_KUBERNETES_TARGET),prev)
KUBERNETES_VERSION := 1.21.2
else ifeq ($(TEST_KUBERNETES_TARGET),prev2)
KUBERNETES_VERSION := 1.20.7
endif
KIND_NODE_IMAGE=kindest/node:v$(KUBERNETES_VERSION)
export TEST_KUBERNETES_VERSION=$(shell echo $(KUBERNETES_VERSION) | grep -o '^[0-9]*\.[0-9]*')
HELM_VALUES_FILE := manifests/values/daemonset-scheduler.yaml
ifeq ($(TEST_SCHEDULER_MANIFEST),deployment)
HELM_VALUES_FILE := manifests/values/deployment-scheduler.yaml
endif
KIND_CONFIG="topolvm-cluster.yaml"
MINIKUBE_FEATURE_GATES=""
MINIKUBE_SCHEDULER_MANIFEST=kube-scheduler.yaml
ifeq ($(KUBERNETES_VERSION),1.22.2)
KIND_CONFIG="topolvm-cluster-single-writer.yaml"
MINIKUBE_FEATURE_GATES="ReadWriteOncePod=true"
MINIKUBE_SCHEDULER_MANIFEST=kube-scheduler-single-writer.yaml
endif
HELM_VALUES_FILE_LVMD := ""
ifeq ($(STORAGE_CAPACITY),true)
ifneq ($(KUBERNETES_VERSION),1.20.7)
HELM_VALUES_FILE_LVMD := manifests/values/daemonset-lvmd-storage-capacity.yaml
HELM_VALUES_FILE := manifests/values/storage-capacity.yaml
else
HELM_VALUES_FILE := ""
endif
else
HELM_VALUES_FILE_LVMD := manifests/values/daemonset-lvmd.yaml
endif
SCHEDULER_CONFIG := scheduler-config-v1beta1-$(TEST_SCHEDULER_MANIFEST).yaml
GO_FILES := $(shell find .. -prune -o -path ../e2e -prune -o -name '*.go' -print)
BACKING_STORE := ./build
topolvm.img: $(GO_FILES)
rm -rf tmpbin
mkdir -p tmpbin
CGO_ENABLED=0 go build -o tmpbin/hypertopolvm ../pkg/hypertopolvm
ln -s hypertopolvm ./tmpbin/lvmd
ln -s hypertopolvm ./tmpbin/topolvm-scheduler
ln -s hypertopolvm ./tmpbin/topolvm-node
ln -s hypertopolvm ./tmpbin/topolvm-controller
$(MAKE) -f ../csi-sidecars.mk OUTPUT_DIR=tmpbin
docker build --no-cache --rm=false -f Dockerfile -t topolvm:dev tmpbin
docker save -o $@ topolvm:dev
/tmp/topolvm/scheduler/scheduler-config.yaml: $(SCHEDULER_CONFIG)
mkdir -p /tmp/topolvm/scheduler
sed -e "s|@DEPLOYMENT_SCHEDULER_HOST@|topolvm-e2e-worker|" $< > $@
.PHONY: launch-kind
launch-kind: /tmp/topolvm/scheduler/scheduler-config.yaml
$(SUDO) rm -rf /tmp/topolvm/controller /tmp/topolvm/worker*
sed -e "s|@KUBERNETES_VERSION@|$(KUBERNETES_VERSION)|" $(KIND_CONFIG) > /tmp/$(KIND_CONFIG)
$(KIND) create cluster --name=$(KIND_CLUSTER_NAME) --config /tmp/$(KIND_CONFIG) --image $(KIND_NODE_IMAGE)
.PHONY: shutdown-kind
shutdown-kind:
$(KIND) delete cluster --name=$(KIND_CLUSTER_NAME) || true
sleep 2
for d in $$($(SUDO) find /tmp/topolvm -type d); do \
if $(SUDO) mountpoint -q $$d; then \
$(SUDO) umount $$d; \
fi; \
done
for d in $$(mount | grep /lib/kubelet | cut -d ' ' -f 3); do $(SUDO) umount $$d; done
.PHONY: start-lvmd
start-lvmd:
mkdir -p build $(BACKING_STORE)
go build -o build/lvmd ../pkg/lvmd
if [ $$(ls -1 $(BACKING_STORE)/backing_store* 2>/dev/null | wc -l) -ne 0 ]; then $(MAKE) stop-lvmd; fi
for i in $$(seq 3); do \
mkdir -p /tmp/topolvm/worker$$i; \
mkdir -p /tmp/topolvm/lvmd$$i; \
truncate --size=20G $(BACKING_STORE)/backing_store$${i}_1; \
$(SUDO) losetup -f $(BACKING_STORE)/backing_store$${i}_1; \
$(SUDO) vgcreate -y node$${i}-myvg1 $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store$${i}_1 | cut -d: -f1); \
$(SUDO) lvcreate -y -n csi-node-test-block -L 1G node$${i}-myvg1; \
$(SUDO) lvcreate -y -n csi-node-test-fs -L 1G node$${i}-myvg1; \
done
# Create additional Volume Groups
truncate --size=10G $(BACKING_STORE)/backing_store1_2; \
$(SUDO) losetup -f $(BACKING_STORE)/backing_store1_2; \
$(SUDO) vgcreate -y node1-myvg2 $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store1_2 | cut -d: -f1); \
truncate --size=10G $(BACKING_STORE)/backing_store2_2; \
$(SUDO) losetup -f $(BACKING_STORE)/backing_store2_2; \
$(SUDO) vgcreate -y node2-myvg2 $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store2_2 | cut -d: -f1); \
truncate --size=10G $(BACKING_STORE)/backing_store3_3; \
$(SUDO) losetup -f $(BACKING_STORE)/backing_store3_3; \
$(SUDO) vgcreate -y node3-myvg3 $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store3_3 | cut -d: -f1); \
for i in $$(seq 3); do \
$(SUDO) systemd-run --unit=lvmd$$i.service $(shell pwd)/build/lvmd --config=$(shell pwd)/lvmd$$i.yaml; \
done
.PHONY: stop-lvmd
stop-lvmd:
$(MAKE) shutdown-kind
for i in $$(seq 3); do \
if systemctl is-active -q lvmd$$i.service; then $(SUDO) systemctl stop lvmd$$i.service; fi; \
for j in $$(seq 3); do \
if [ -f $(BACKING_STORE)/backing_store$${i}_$${j} ]; then \
$(SUDO) vgremove -ffy node$${i}-myvg$${j}; \
$(SUDO) pvremove -ffy $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store$${i}_$${j} | cut -d: -f1); \
$(SUDO) losetup -d $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store$${i}_$${j} | cut -d: -f1); \
rm -f $(BACKING_STORE)/backing_store$${i}_$${j}; \
fi; \
done; \
done
.PHONY: test
test: topolvm.img
ifneq ($(HELM_VALUES_FILE),"")
$(MAKE) shutdown-kind
$(MAKE) launch-kind
$(KIND) load image-archive --name=$(KIND_CLUSTER_NAME) topolvm.img
$(KUBECTL) apply -f https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.crds.yaml
$(KUBECTL) create namespace topolvm-system
$(KUBECTL) label namespace topolvm-system topolvm.cybozu.com/webhook=ignore
$(HELM) repo add jetstack https://charts.jetstack.io
$(HELM) repo update
$(HELM) dependency build ../charts/topolvm/
$(HELM) install --namespace=topolvm-system topolvm ../charts/topolvm/ -f $(HELM_VALUES_FILE)
$(KUBECTL) apply -f manifests/common/
$(SUDO) -E env PATH=${PATH} E2ETEST=1 BINDIR=$(BINDIR) STORAGE_CAPACITY=$(STORAGE_CAPACITY) READ_WRITE_ONCE_POD=$(READ_WRITE_ONCE_POD) GOLANG_PROTOBUF_REGISTRATION_CONFLICT=warn $(GINKGO) --failFast -v .
endif
.PHONY: clean
clean: stop-lvmd
rm -rf \
topolvm.img \
build/ \
$(BACKING_STORE)/backing_store* \
/tmp/topolvm/scheduler/scheduler-config.yaml
.PHONY: setup
setup:
cd ..; $(MAKE) setup
mkdir -p $(BINDIR)
curl -o $(BINDIR)/kubectl -sfL https://storage.googleapis.com/kubernetes-release/release/v$(KUBERNETES_VERSION)/bin/linux/amd64/kubectl
chmod a+x $(BINDIR)/kubectl
.PHONY: daemonset-lvmd/create-vg
daemonset-lvmd/create-vg:
mkdir -p build $(BACKING_STORE)
if [ $$(ls -1 $(BACKING_STORE)/backing_store_lvmd* 2>/dev/null | wc -l) -ne 0 ]; then $(MAKE) $(@D)/remove-vg; fi
for i in $$(seq 3); do \
truncate --size=20G $(BACKING_STORE)/backing_store_lvmd_$${i}; \
$(SUDO) losetup -f $(BACKING_STORE)/backing_store_lvmd_$${i}; \
$(SUDO) vgcreate -y node-myvg$${i} $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store_lvmd_$${i} | cut -d: -f1); \
$(SUDO) lvcreate -y -n csi-node-test-block -L 1G node-myvg$${i}; \
$(SUDO) lvcreate -y -n csi-node-test-fs -L 1G node-myvg$${i}; \
done
.PHONY: daemonset-lvmd/remove-vg
daemonset-lvmd/remove-vg:
for i in $$(seq 3); do \
$(SUDO) vgremove -ffy node-myvg$${i}; \
$(SUDO) pvremove -ffy $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store_lvmd_$${i} | cut -d: -f1); \
$(SUDO) losetup -d $$($(SUDO) losetup -j $(BACKING_STORE)/backing_store_lvmd_$${i} | cut -d: -f1); \
rm -f $(BACKING_STORE)/backing_store_lvmd_$${i}; \
done
.PHONY: daemonset-lvmd/setup-minikube
daemonset-lvmd/setup-minikube:
mkdir -p $(BINDIR)
$(SUDO) apt-get update
DEBIAN_FRONTEND=noninteractive $(SUDO) apt-get install -y --no-install-recommends conntrack
curl -LO https://github.com/kubernetes/minikube/releases/download/$(MINIKUBE_VERSION)/minikube-linux-amd64
chmod a+x minikube-linux-amd64
mv minikube-linux-amd64 $(BINDIR)/minikube
.PHONY: daemonset-lvmd/launch-minikube
daemonset-lvmd/launch-minikube:
$(SUDO) -E $(BINDIR)/minikube start \
--vm-driver=none \
--kubernetes-version=v$(KUBERNETES_VERSION) \
--extra-config=kubelet.read-only-port=10255 \
--feature-gates=$(MINIKUBE_FEATURE_GATES)
$(SUDO) chown -R $$USER $$HOME/.kube $(MINIKUBE_HOME)/.minikube
$(SUDO) chmod -R a+r $$HOME/.kube $(MINIKUBE_HOME)/.minikube
$(SUDO) find $(MINIKUBE_HOME)/.minikube -name id_rsa -exec chmod 600 {} ';'
.PHONY: daemonset-lvmd/delete-minikube
daemonset-lvmd/delete-minikube:
$(SUDO) -E $(BINDIR)/minikube delete || true
# Set scheduler configs manually because minikube can't edit scheduler configs.
.PHONY: daemonset-lvmd/update-minikube-setting
daemonset-lvmd/update-minikube-setting: daemonset-lvmd/delete-minikube daemonset-lvmd/launch-minikube
ifneq ($(STORAGE_CAPACITY),true)
$(SUDO) mkdir -p /var/lib/scheduler
$(SUDO) sh -c 'sed -e "s|@DEPLOYMENT_SCHEDULER_HOST@|127.0.0.1|" scheduler-config-v1beta1-deployment.yaml > /var/lib/scheduler/scheduler-config.yaml'
$(SUDO) sh -c 'sed -e "s|@KUBERNETES_VERSION@|$(KUBERNETES_VERSION)|" $(MINIKUBE_SCHEDULER_MANIFEST) > /etc/kubernetes/manifests/kube-scheduler.yaml'
endif
.PHONY: daemonset-lvmd/test
daemonset-lvmd/test: topolvm.img
ifneq ($(HELM_VALUES_FILE_LVMD),"")
$(KUBECTL) apply -f https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.crds.yaml
$(KUBECTL) create namespace topolvm-system
$(KUBECTL) label namespace topolvm-system topolvm.cybozu.com/webhook=ignore
$(HELM) repo add jetstack https://charts.jetstack.io
$(HELM) repo update
$(HELM) dependency build ../charts/topolvm/
$(HELM) install --create-namespace --namespace=topolvm-system topolvm ../charts/topolvm/ -f $(HELM_VALUES_FILE_LVMD)
$(KUBECTL) apply -f manifests/common/
$(SUDO) -E env PATH=${PATH} E2ETEST=1 BINDIR=$(BINDIR) STORAGE_CAPACITY=$(STORAGE_CAPACITY) READ_WRITE_ONCE_POD=$(READ_WRITE_ONCE_POD) DAEMONSET_LVMD=true GOLANG_PROTOBUF_REGISTRATION_CONFLICT=warn $(GINKGO) --failFast -v .
endif
.PHONY: daemonset-lvmd/clean
daemonset-lvmd/clean: daemonset-lvmd/delete-minikube daemonset-lvmd/remove-vg