-
Notifications
You must be signed in to change notification settings - Fork 99
/
dev-up.sh
executable file
·176 lines (145 loc) · 7.13 KB
/
dev-up.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#!/usr/bin/env bash
set -ueo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
if [[ -n "${DEBUG:-}" ]]; then set -x; fi
# Redirect all stdout to stderr.
{
if ! hcloud version >/dev/null; then echo "ERROR: 'hcloud' CLI not found, please install it and make it available on your \$PATH"; exit 1; fi
if ! k3sup version >/dev/null; then echo "ERROR: 'k3sup' not found, please install it and make it available on your \$PATH"; exit 1; fi
if ! helm version >/dev/null; then echo "ERROR: 'helm' not found, please install it and make it available on your \$PATH"; exit 1; fi
if [[ "${HCLOUD_TOKEN:-}" == "" ]]; then echo "ERROR: please set \$HCLOUD_TOKEN"; exit 1; fi
# We run a lot of subshells below for speed. If any encounter an error, we shut down the whole process group, pronto.
function error() {
echo "Onoes, something went wrong! :( The output above might have some clues."
kill 0
}
trap error ERR
image_name=${IMAGE_NAME:-ubuntu-20.04}
instance_count=${INSTANCES:-3}
instance_type=${INSTANCE_TYPE:-cpx11}
location=${LOCATION:-fsn1}
network_zone=${NETWORK_ZONE:-eu-central}
ssh_keys=${SSH_KEYS:-}
# All k3s after January 2024 break our e2e tests, we hardcode
# the versions for now until we can fix the source of this.
# channel=${K3S_CHANNEL:-stable}
k3s_version=${K3S_VERSION:-v1.28.5+k3s1}
network_cidr=${NETWORK_CIDR:-10.0.0.0/8}
subnet_cidr=${SUBNET_CIDR:-10.0.0.0/24}
cluster_cidr=${CLUSTER_CIDR:-10.244.0.0/16}
scope="${SCOPE:-dev}"
scope=${scope//[^a-zA-Z0-9_]/-}
scope_name=csi-${scope}
label="managedby=hack,scope=$scope_name"
ssh_private_key="$SCRIPT_DIR/.ssh-$scope"
k3s_opts=${K3S_OPTS:-"--kubelet-arg cloud-provider=external"}
k3s_server_opts=${K3S_SERVER_OPTS:-"--disable-cloud-controller --disable=traefik --disable=servicelb --disable=local-storage --flannel-backend=none --cluster-cidr ${cluster_cidr}"}
echo -n "$HCLOUD_TOKEN" > "$SCRIPT_DIR/.token-$scope"
export KUBECONFIG="$SCRIPT_DIR/.kubeconfig-$scope"
ssh_command="ssh -i $ssh_private_key -o StrictHostKeyChecking=off -o BatchMode=yes -o ConnectTimeout=5"
# Generate SSH keys and upload publkey to Hetzner Cloud.
( trap error ERR
[[ ! -f $ssh_private_key ]] && ssh-keygen -t ed25519 -f $ssh_private_key -C '' -N ''
[[ ! -f $ssh_private_key.pub ]] && ssh-keygen -y -f $ssh_private_key > $ssh_private_key.pub
if ! hcloud ssh-key describe $scope_name >/dev/null 2>&1; then
hcloud ssh-key create --label $label --name $scope_name --public-key-from-file $ssh_private_key.pub
fi
) &
# Create Network
( trap error ERR
if ! hcloud network describe $scope_name >/dev/null 2>&1; then
hcloud network create --label $label --ip-range $network_cidr --name $scope_name
hcloud network add-subnet --network-zone $network_zone --type cloud --ip-range $subnet_cidr $scope_name
fi
) &
for num in $(seq $instance_count); do
# Create server and initialize Kubernetes on it with k3sup.
( trap error ERR
server_name="$scope_name-$num"
# Maybe cluster is already up and node is already there.
if kubectl get node $server_name >/dev/null 2>&1; then
exit 0
fi
ip=$(hcloud server ip $server_name 2>/dev/null || true)
if [[ -z "${ip:-}" ]]; then
# Wait for SSH key
until hcloud ssh-key describe $scope_name >/dev/null 2>&1; do sleep 1; done
until hcloud network describe $scope_name 2>&1 | grep $subnet_cidr >/dev/null; do sleep 1; done
createcmd="hcloud server create --image $image_name --label $label --location $location --name $server_name --ssh-key=$scope_name --type $instance_type --network $scope_name"
for key in $ssh_keys; do
createcmd+=" --ssh-key $key"
done
$createcmd
ip=$(hcloud server ip $server_name)
fi
# Wait for SSH.
until [ "$($ssh_command root@$ip echo ok 2>/dev/null)" = "ok" ]; do
sleep 1
done
$ssh_command root@$ip 'mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml' < $SCRIPT_DIR/k3s-registries.yaml
private_ip=$(hcloud server describe $server_name -o format="{{ (index .PrivateNet 0).IP }}")
k3s_node_ip_opts="--node-external-ip ${ip} --node-ip ${private_ip}"
if [[ "$num" == "1" ]]; then
# First node is control plane.
k3sup install --print-config=false --ip $ip --k3s-version "${k3s_version}" --k3s-extra-args "${k3s_server_opts} ${k3s_opts} ${k3s_node_ip_opts}" --local-path $KUBECONFIG --ssh-key $ssh_private_key
else
# All subsequent nodes are initialized as workers.
# Can't go any further until control plane has bootstrapped a bit though.
until $ssh_command root@$(hcloud server ip $scope_name-1 || true) stat /etc/rancher/node/password >/dev/null 2>&1; do
sleep 1
done
k3sup join --server-ip $(hcloud server ip $scope_name-1) --ip $ip --k3s-version "${k3s_version}" --k3s-extra-args "${k3s_opts} ${k3s_node_ip_opts}" --ssh-key $ssh_private_key
fi
) &
# Wait for this node to show up in the cluster.
( trap error ERR; set +x
until kubectl wait --for=condition=Ready node/$scope_name-$num >/dev/null 2>&1; do sleep 1; done
echo $scope_name-$num is up and in cluster
) &
done
( trap error ERR
# Control plane init tasks.
# This is running in parallel with the server init, above.
# Wait for control plane to look alive.
until kubectl get nodes >/dev/null 2>&1; do sleep 1; done;
# Deploy private registry.
( trap error ERR
if ! helm status -n kube-system registry >/dev/null 2>&1; then
helm install registry docker-registry \
--repo=https://helm.twun.io \
-n kube-system \
--version 2.2.2 \
--set service.clusterIP=10.43.0.2 \
--set 'tolerations[0].key=node.cloudprovider.kubernetes.io/uninitialized' \
--set 'tolerations[0].operator=Exists'
fi
) &
# Install Cilium.
( trap error ERR
if ! helm status -n kube-system cilium >/dev/null 2>&1; then
helm install cilium cilium --repo https://helm.cilium.io/ -n kube-system --version 1.13.1 \
--set tunnel=disabled \
--set ipv4NativeRoutingCIDR=$cluster_cidr \
--set ipam.mode=kubernetes
fi) &
# Create HCLOUD_TOKEN Secret for hcloud-cloud-controller-manager.
( trap error ERR
if ! kubectl -n kube-system get secret hcloud >/dev/null 2>&1; then
kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN" --from-literal="network=$scope_name"
fi) &
wait
# Install hcloud-cloud-controller-manager
( trap error ERR
if ! helm status -n kube-system hccm >/dev/null 2>&1; then
helm install hccm hcloud-cloud-controller-manager --repo https://charts.hetzner.cloud/ -n kube-system --version 1.14.2 --set networking.enabled=true
fi) &
wait
) &
wait
echo "Success - cluster fully initialized and ready, why not see for yourself?"
echo '$ kubectl get nodes'
kubectl get nodes
} >&2
echo "export KUBECONFIG=$KUBECONFIG"
$SCRIPT_DIR/registry-port-forward.sh
echo "export SKAFFOLD_DEFAULT_REPO=localhost:30666"