-
Notifications
You must be signed in to change notification settings - Fork 0
/
install.sh
219 lines (191 loc) · 8.15 KB
/
install.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#!/bin/bash
set -e
export user=$(whoami)
sudo mkdir -p /opt/k8s/{registry,controller/pg/data}
sudo chmod g+s /opt/k8s
sudo setfacl -m d:g::rwX /opt/k8s
sudo setfacl -m g::rwX /opt/k8s
sudo setfacl -m d:u::rwX /opt/k8s
sudo setfacl -m u::rwX /opt/k8s
sudo setfacl -m d:u:${user}:rwX /opt/k8s
sudo setfacl -m u:${user}:rwX /opt/k8s
sudo setfacl -m d:o::- /opt/k8s
sudo setfacl -m o::- /opt/k8s
cat <<'EOF' >"/opt/k8s/node-prep.sh"
#!/bin/bash
function prepareNode() {
apt update && apt install -y apt-transport-https curl docker.io
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > "/etc/apt/sources.list.d/kubernetes.list"
sysctl net.bridge.bridge-nf-call-iptables=1
systemctl enable docker.service
cat > /etc/docker/daemon.json <<DOCKEREOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "10m",
"max-file": "2"
},
"storage-driver": "overlay2"
}
DOCKEREOF
systemctl restart docker
systemctl restart systemd-networkd
apt update
apt install -y kubelet kubeadm kubectl
systemctl daemon-reload
systemctl restart kubelet
}
export -f prepareNode
EOF
sudo chmod +x /opt/k8s/node-prep.sh
source /opt/k8s/node-prep.sh
sudo su -c "$(declare -f prepareNode); prepareNode"
export HOSTNAME=$(hostname)
export IPS=($(hostname -I))
if [ "${KUBE_LB_HOST}" == "" ]; then
export KUBE_LB_HOST=${IPS[0]}
fi
cat <<EOF >"/opt/k8s/kubeadm-config.yaml"
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
serviceSubnet: 10.0.0.0/12
podSubnet: 10.16.0.0/12
apiServer:
extraArgs:
service-account-issuer: kubernetes.default.svc
service-account-signing-key-file: /etc/kubernetes/pki/sa.key
service-account-api-audiences: 'true'
certSANs:
- 127.0.0.1
- localhost
- ${HOSTNAME}
EOF
for ip in ${IPS[@]}; do
cat <<EOF >>"/opt/k8s/kubeadm-config.yaml"
- "${ip}"
EOF
done
cat <<EOF >>"/opt/k8s/kubeadm-config.yaml"
- "${KUBE_LB_HOST}"
controlPlaneEndpoint: "${KUBE_LB_HOST}:6443"
EOF
sudo kubeadm init --node-name ${HOSTNAME} --skip-token-print --skip-certificate-key-print --config /opt/k8s/kubeadm-config.yaml
mkdir -p ${HOME}/.kube
sudo cp -f /etc/kubernetes/admin.conf ${HOME}/.kube/config
sudo chown -R ${user}:${user} ${HOME}/.kube
kubectl apply -f https://raw.githubusercontent.com/djgilcrease/kubeadm-scripts/master/kube-flannel.yml
kubectl taint nodes ${HOSTNAME} node-role.kubernetes.io/master-
kubectl wait --for=condition=ready node ${HOSTNAME} --timeout=120s
echo 'k8s is running in single-machine mode.'
echo
echo 'To add additional control nodes run'
echo '/opt/k8s/new-controller-join.sh $(whoami) <NEW_CONTROLER_IP> $(kubeadm token create --ttl 3m)'
echo
echo 'To add additional control nodes run'
echo '/opt/k8s/new-worker-join.sh $(whoami) <NEW_WORKER_IP> $(kubeadm token create --ttl 3m)'
echo
echo 'Adding additional nodes will remove the single-machine mode.'
echo
################################
# Scripts to manage other nodes
################################
export KUBE_DISCOVERY_TOKEN_HASH=$(sudo openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
cat <<EOF >"/opt/k8s/node.env"
export KUBE_LB_HOST=${KUBE_LB_HOST}
export KUBE_DISCOVERY_TOKEN_HASH=${KUBE_DISCOVERY_TOKEN_HASH}
EOF
cat <<'EOF' >"/opt/k8s/new-controller-join.sh"
#!/bin/bash
user=$1
host=$2
token=$3
set +e
kubectl taint nodes -l node-role.kubernetes.io/master node-role.kubernetes.io/master=true:NoSchedule
set -e
mkdir -p /tmp/ncn/
sudo cp -R /etc/kubernetes/pki /tmp/ncn/.
sudo cp /etc/kubernetes/admin.conf /tmp/ncn/.
sudo chown -R ${user}:${user} /tmp/ncn/
scp /tmp/ncn/pki/ca.crt ${user}@${host}:
scp /tmp/ncn/pki/ca.key ${user}@${host}:
scp /tmp/ncn/pki/sa.key ${user}@${host}:
scp /tmp/ncn/pki/sa.pub ${user}@${host}:
scp /tmp/ncn/pki/front-proxy-ca.crt ${user}@${host}:
scp /tmp/ncn/pki/front-proxy-ca.key ${user}@${host}:
scp /tmp/ncn/pki/etcd/ca.crt ${user}@${host}:etcd-ca.crt
scp /tmp/ncn/pki/etcd/ca.key ${user}@${host}:etcd-ca.key
scp /tmp/ncn/admin.conf ${user}@${host}:
sudo rm -rf /tmp/ncn
scp /opt/k8s/node-prep.sh ${user}@${host}:
scp /opt/k8s/new-worker-join.sh ${user}@${host}:
scp /opt/k8s/new-controller-join.sh ${user}@${host}:
scp /opt/k8s/node.env ${user}@${host}:
ssh ${user}@${host} "sudo mkdir -p /opt/k8s/"
ssh ${user}@${host} "sudo setfacl -m d:g::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m g::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:u::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m u::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:u:${user}:rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m u:${user}:rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:o::- /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m o::- /opt/k8s"
ssh ${user}@${host} "sudo mv /home/${user}/node-prep.sh /opt/k8s/"
ssh ${user}@${host} "sudo mv /home/${user}/node.env /opt/k8s/"
ssh ${user}@${host} "sudo mv /home/${user}/new-worker-join.sh /opt/k8s/"
ssh ${user}@${host} "sudo mv /home/${user}/new-controller-join.sh /opt/k8s/"
ssh ${user}@${host} 'source /opt/k8s/node-prep.sh && sudo su -c "$(declare -f prepareNode); prepareNode"'
ssh ${user}@${host} "sudo mkdir -p /etc/kubernetes/pki/etcd"
ssh ${user}@${host} "sudo mv /home/${user}/ca.crt /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/ca.key /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/sa.pub /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/sa.key /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/front-proxy-ca.crt /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/front-proxy-ca.key /etc/kubernetes/pki/"
ssh ${user}@${host} "sudo mv /home/${user}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt"
ssh ${user}@${host} "sudo mv /home/${user}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key"
ssh ${user}@${host} "sudo chown -R root:root /etc/kubernetes/"
ssh ${user}@${host} "echo 'export token=${token}' >> /opt/k8s/node.env"
ssh ${user}@${host} 'source /opt/k8s/node.env && sudo kubeadm join ${KUBE_LB_HOST}:6443 --token ${token} --discovery-token-ca-cert-hash sha256:${KUBE_DISCOVERY_TOKEN_HASH} --experimental-control-plane'
ssh ${user}@${host} "sed -i '$ d' /opt/k8s/node.env"
ssh ${user}@${host} "mkdir -p $HOME/.kube"
ssh ${user}@${host} "sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config"
ssh ${user}@${host} "sudo chown ${user}:${user} $HOME/.kube/config"
ssh ${user}@${host} "kubectl wait --for=condition=ready node $(hostname) --timeout=120s"
EOF
sudo chmod +x /opt/k8s/new-controller-join.sh
cat <<'EOF' >"/opt/k8s/new-worker-join.sh"
#!/bin/bash
user=$1
host=$2
token=$3
set +e
kubectl taint nodes -l node-role.kubernetes.io/master node-role.kubernetes.io/master=true:NoSchedule
set -e
mkdir -p /tmp/nwn/
sudo cp /etc/kubernetes/admin.conf /tmp/nwn/.
sudo chown -R ${user}:${user} /tmp/nwn/
scp /tmp/nwn/admin.conf ${user}@${host}:
scp /opt/k8s/node-prep.sh ${user}@${host}:
scp /opt/k8s/node.env ${user}@${host}:
sudo rm -rf /tmp/nwn
ssh ${user}@${host} "sudo mkdir -p /opt/k8s/"
ssh ${user}@${host} "sudo setfacl -m d:g::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m g::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:u::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m u::rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:u:${user}:rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m u:${user}:rwX /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m d:o::- /opt/k8s"
ssh ${user}@${host} "sudo setfacl -m o::- /opt/k8s"
ssh ${user}@${host} "sudo mv /home/${user}/node-prep.sh /opt/k8s/"
ssh ${user}@${host} "sudo mv /home/${user}/node.env /opt/k8s/"
ssh ${user}@${host} 'source /opt/k8s/node-prep.sh && sudo su -c "$(declare -f prepareNode); prepareNode"'
ssh ${user}@${host} "echo 'export token=${token}' >> /opt/k8s/node.env"
ssh ${user}@${host} 'source /opt/k8s/node.env && sudo kubeadm join ${KUBE_LB_HOST}:6443 --token ${token} --discovery-token-ca-cert-hash sha256:${KUBE_DISCOVERY_TOKEN_HASH}'
ssh ${user}@${host} "sudo mv /home/${user}/admin.conf $HOME/.kube/config"
ssh ${user}@${host} "sudo rm -rf /opt/k8s"
EOF
sudo chmod +x /opt/k8s/new-worker-join.sh