From 9c94891aa03b1d6e03d09992b6c4bfd5fda2169c Mon Sep 17 00:00:00 2001
From: Foivos Filippopoulos <foivosfilip@gmail.com>
Date: Wed, 26 Jan 2022 16:32:31 +0000
Subject: [PATCH] Flag to allow using deprecated docker runtime

Helps to mitigate the inotify fd leak issue related with containerd runtime:
https://github.com/containerd/containerd/issues/5670
Created as a flag to be able to support both docker and containerd runtime
setups
---
 common.tf                           | 10 +++++++++-
 master.tf                           | 13 ++++++++-----
 modules/cert-refresh-master/main.tf | 14 ++++++++++----
 modules/cert-refresh-node/main.tf   |  8 +++++++-
 node-common.tf                      |  6 ++++--
 resources/docker-dropin.conf        |  4 ++++
 resources/master-kubelet-conf.yaml  |  2 +-
 resources/master-kubelet.service    |  9 +++++++++
 resources/node-kubelet-conf.yaml    |  2 +-
 resources/node-kubelet.service      |  9 +++++++++
 variables.tf                        |  6 ++++++
 worker.tf                           | 11 ++++++-----
 12 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/common.tf b/common.tf
index a61b4ba..cefb632 100644
--- a/common.tf
+++ b/common.tf
@@ -25,12 +25,20 @@ data "ignition_file" "cfssljson" {
   }
 }
 
+data "template_file" "docker_opts_dropin" {
+  template = file("${path.module}/resources/docker-dropin.conf")
+
+  vars = {
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
+  }
+}
+
 data "ignition_systemd_unit" "docker-opts-dropin" {
   name = "docker.service"
 
   dropin {
     name    = "10-custom-options.conf"
-    content = file("${path.module}/resources/docker-dropin.conf")
+    content = data.template_file.docker_opts_dropin.rendered
   }
 }
 
diff --git a/master.tf b/master.tf
index 43bc05b..8eaa796 100644
--- a/master.tf
+++ b/master.tf
@@ -4,8 +4,9 @@ data "ignition_systemd_unit" "locksmithd_master" {
 }
 
 module "cert-refresh-master" {
-  source      = "./modules/cert-refresh-master"
-  on_calendar = var.cfssl_node_renew_timer
+  source                        = "./modules/cert-refresh-master"
+  on_calendar                   = var.cfssl_node_renew_timer
+  use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
 }
 
 // Node certificate for kubelet to use as part of system:master-nodes. We need
@@ -216,9 +217,10 @@ data "template_file" "master-kubelet" {
   template = file("${path.module}/resources/master-kubelet.service")
 
   vars = {
-    kubelet_binary_path = "/opt/bin/kubelet"
-    cloud_provider      = var.cloud_provider
-    get_hostname        = var.node_name_command[var.cloud_provider]
+    kubelet_binary_path           = "/opt/bin/kubelet"
+    cloud_provider                = var.cloud_provider
+    get_hostname                  = var.node_name_command[var.cloud_provider]
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
   }
 }
 
@@ -234,6 +236,7 @@ data "template_file" "master-kubelet-conf" {
     cluster_dns                       = local.cluster_dns_yaml
     feature_gates                     = local.feature_gates_yaml_fragment
     kubelet_cgroup_v2_runtime_enabled = var.kubelet_cgroup_v2_runtime_enabled
+    use_deprecated_docker_runtime     = var.use_deprecated_docker_runtime
   }
 }
 
diff --git a/modules/cert-refresh-master/main.tf b/modules/cert-refresh-master/main.tf
index 93a8eb3..6b7259c 100644
--- a/modules/cert-refresh-master/main.tf
+++ b/modules/cert-refresh-master/main.tf
@@ -2,13 +2,19 @@ variable "on_calendar" {
   type = string
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 data "ignition_systemd_unit" "cert-refresh" {
   name = "cert-refresh.service"
 
   content = <<EOS
 [Unit]
 Description=Fetch new certificates from cfssl server and restart components to reload certs
-Requires=containerd.service prepare-crictl.service
+${var.use_deprecated_docker_runtime ? "Requires=docker.service" : "Requires=containerd.service prepare-crictl.service" }
 After=network-online.target
 [Service]
 Type=oneshot
@@ -21,9 +27,9 @@ ExecStart=/opt/bin/cfssl-new-scheduler-cert
 ExecStart=/opt/bin/cfssl-new-controller-manager-cert
 # Hack to reload certs on control plane tier
 #  https://github.com/kubernetes/kubernetes/issues/46287
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-controller-manager)"
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-apiserver)"
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-scheduler)"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-controller-manager" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-controller-manager)"}"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-apiserver)" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-apiserver)"}"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-scheduler)" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-scheduler)"}"
 ExecStart=/usr/bin/systemctl try-restart kubelet.service
 Restart=on-failure
 RestartSec=10
diff --git a/modules/cert-refresh-node/main.tf b/modules/cert-refresh-node/main.tf
index fb4dd20..201b317 100644
--- a/modules/cert-refresh-node/main.tf
+++ b/modules/cert-refresh-node/main.tf
@@ -2,13 +2,19 @@ variable "on_calendar" {
   type = string
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 data "ignition_systemd_unit" "cert-refresh" {
   name = "cert-refresh.service"
 
   content = <<EOS
 [Unit]
 Description=Fetch new certificates from cfssl server and restart components to reload certs
-Requires=containerd.service
+${var.use_deprecated_docker_runtime ? "Requires=docker.service" : "Requires=containerd.service" }
 After=network-online.target
 [Service]
 Type=oneshot
diff --git a/node-common.tf b/node-common.tf
index a7634ba..2456642 100644
--- a/node-common.tf
+++ b/node-common.tf
@@ -84,6 +84,7 @@ data "template_file" "node-kubelet-conf" {
     kubelet_cgroup_v2_runtime_enabled = var.kubelet_cgroup_v2_runtime_enabled
     system_reserved_cpu               = var.system_reserved_cpu
     system_reserved_memory            = var.system_reserved_memory
+    use_deprecated_docker_runtime     = var.use_deprecated_docker_runtime
   }
 }
 
@@ -138,6 +139,7 @@ data "ignition_file" "prometheus-ro-rootfs" {
 }
 
 module "cert-refresh-node" {
-  source      = "./modules/cert-refresh-node"
-  on_calendar = var.cfssl_node_renew_timer
+  source                        = "./modules/cert-refresh-node"
+  on_calendar                   = var.cfssl_node_renew_timer
+  use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
 }
diff --git a/resources/docker-dropin.conf b/resources/docker-dropin.conf
index 524dd03..be0bba1 100644
--- a/resources/docker-dropin.conf
+++ b/resources/docker-dropin.conf
@@ -1,2 +1,6 @@
 [Service]
+%{ if use_deprecated_docker_runtime ~}
+Environment=DOCKER_OPTS="--log-opt max-size=100m --log-opt max-file=1"
+%{ else ~}
 Environment=DOCKER_OPTS="--containerd=/run/containerd/containerd.sock --log-opt max-size=100m --log-opt max-file=1"
+%{ endif ~}
diff --git a/resources/master-kubelet-conf.yaml b/resources/master-kubelet-conf.yaml
index 24506e8..64ba5ea 100644
--- a/resources/master-kubelet-conf.yaml
+++ b/resources/master-kubelet-conf.yaml
@@ -10,7 +10,7 @@ authentication:
     clientCAFile: "/etc/kubernetes/ssl/ca.pem"
 authorization:
   mode: AlwaysAllow
-%{ if kubelet_cgroup_v2_runtime_enabled }
+%{ if (kubelet_cgroup_v2_runtime_enabled || use_deprecated_docker_runtime) }
 cgroupDriver: systemd
 %{ endif ~}
 clusterDNS:${cluster_dns}
diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service
index 3f8e72e..4b58e03 100644
--- a/resources/master-kubelet.service
+++ b/resources/master-kubelet.service
@@ -1,7 +1,12 @@
 [Unit]
 Description=Kubernetes Kubelet
+%{ if use_deprecated_docker_runtime ~}
+Requires=docker.service
+After=docker.service
+%{ else ~}
 Requires=containerd.service
 After=containerd.service
+%{ endif ~}
 [Service]
 EnvironmentFile=-/etc/kubernetes/config/kubeletenv
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/config
@@ -21,8 +26,12 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 #  Flag --network-plugin has been deprecated, will be removed along with dockershim.
 ExecStart=${kubelet_binary_path} \
   --config=/etc/kubernetes/config/master-kubelet-conf.yaml \
+%{ if use_deprecated_docker_runtime ~}
+  --container-runtime=docker \
+%{ else ~}
   --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
   --container-runtime=remote \
+%{ endif ~}
   --exit-on-lock-contention \
   --hostname-override="$${NODE_HOSTNAME}" \
   --kubeconfig=/var/lib/kubelet/kubeconfig \
diff --git a/resources/node-kubelet-conf.yaml b/resources/node-kubelet-conf.yaml
index 6f1a7b4..87333c9 100644
--- a/resources/node-kubelet-conf.yaml
+++ b/resources/node-kubelet-conf.yaml
@@ -10,7 +10,7 @@ authentication:
     clientCAFile: "/etc/kubernetes/ssl/ca.pem"
 authorization:
   mode: AlwaysAllow
-%{ if kubelet_cgroup_v2_runtime_enabled }
+%{ if (kubelet_cgroup_v2_runtime_enabled || use_deprecated_docker_runtime)}
 cgroupDriver: systemd
 %{ endif ~}
 clusterDNS:${cluster_dns}
diff --git a/resources/node-kubelet.service b/resources/node-kubelet.service
index afa6708..2d2be4a 100644
--- a/resources/node-kubelet.service
+++ b/resources/node-kubelet.service
@@ -1,7 +1,12 @@
 [Unit]
 Description=Kubernetes Kubelet
+%{ if use_deprecated_docker_runtime ~}
+Requires=docker.service
+After=docker.service
+%{ else ~}
 Requires=containerd.service
 After=containerd.service
+%{ endif ~}
 [Service]
 EnvironmentFile=-/etc/kubernetes/config/kubeletenv
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/config
@@ -17,8 +22,12 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 # args below `--v=0` are deprecated
 ExecStart=${kubelet_binary_path} \
   --config=/etc/kubernetes/config/node-kubelet-conf.yaml \
+%{ if use_deprecated_docker_runtime ~}
+  --container-runtime=docker \
+%{ else ~}
   --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
   --container-runtime=remote \
+%{ endif ~}
   --exit-on-lock-contention \
   --hostname-override="$${NODE_HOSTNAME}" \
   --kubeconfig=/var/lib/kubelet/kubeconfig \
diff --git a/variables.tf b/variables.tf
index a21cec0..5489f93 100644
--- a/variables.tf
+++ b/variables.tf
@@ -265,6 +265,12 @@ variable "containerd_no_shim" {
   type        = bool
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 locals {
   # Comma separated list for cli flas use, example output:
   # `ExpandPersistentVolumes=true,PodShareProcessNamespace=true,AdvancedAuditing=false`
diff --git a/worker.tf b/worker.tf
index 82effba..7fbc539 100644
--- a/worker.tf
+++ b/worker.tf
@@ -7,11 +7,12 @@ data "template_file" "worker-kubelet" {
   template = file("${path.module}/resources/node-kubelet.service")
 
   vars = {
-    kubelet_binary_path = "/opt/bin/kubelet"
-    cloud_provider      = var.cloud_provider
-    get_hostname        = var.node_name_command[var.cloud_provider]
-    labels              = "role=worker"
-    taints              = ""
+    kubelet_binary_path           = "/opt/bin/kubelet"
+    cloud_provider                = var.cloud_provider
+    get_hostname                  = var.node_name_command[var.cloud_provider]
+    labels                        = "role=worker"
+    taints                        = ""
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
   }
 }