Flag to allow using deprecated docker runtime

Helps to mitigate the inotify fd leak issue related with containerd runtime:
containerd/containerd#5670
Created as a flag to be able to support both docker and containerd runtime
setups

common.tf

@@ -25,12 +25,20 @@ data "ignition_file" "cfssljson" {
   }
 }
 
+data "template_file" "docker_opts_dropin" {
+  template = file("${path.module}/resources/docker-dropin.conf")
+
+  vars = {
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
+  }
+}
+
 data "ignition_systemd_unit" "docker-opts-dropin" {
   name = "docker.service"
 
   dropin {
     name    = "10-custom-options.conf"
-    content = file("${path.module}/resources/docker-dropin.conf")
+    content = data.template_file.docker_opts_dropin.rendered
   }
 }
 

master.tf

@@ -4,8 +4,9 @@ data "ignition_systemd_unit" "locksmithd_master" {
 }
 
 module "cert-refresh-master" {
-  source      = "./modules/cert-refresh-master"
-  on_calendar = var.cfssl_node_renew_timer
+  source                        = "./modules/cert-refresh-master"
+  on_calendar                   = var.cfssl_node_renew_timer
+  use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
 }
 
 // Node certificate for kubelet to use as part of system:master-nodes. We need
@@ -216,9 +217,10 @@ data "template_file" "master-kubelet" {
   template = file("${path.module}/resources/master-kubelet.service")
 
   vars = {
-    kubelet_binary_path = "/opt/bin/kubelet"
-    cloud_provider      = var.cloud_provider
-    get_hostname        = var.node_name_command[var.cloud_provider]
+    kubelet_binary_path           = "/opt/bin/kubelet"
+    cloud_provider                = var.cloud_provider
+    get_hostname                  = var.node_name_command[var.cloud_provider]
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
   }
 }
 
@@ -234,6 +236,7 @@ data "template_file" "master-kubelet-conf" {
     cluster_dns                       = local.cluster_dns_yaml
     feature_gates                     = local.feature_gates_yaml_fragment
     kubelet_cgroup_v2_runtime_enabled = var.kubelet_cgroup_v2_runtime_enabled
+    use_deprecated_docker_runtime     = var.use_deprecated_docker_runtime
   }
 }
 

modules/cert-refresh-master/main.tf

@@ -2,13 +2,19 @@ variable "on_calendar" {
   type = string
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 data "ignition_systemd_unit" "cert-refresh" {
   name = "cert-refresh.service"
 
   content = <<EOS
 [Unit]
 Description=Fetch new certificates from cfssl server and restart components to reload certs
-Requires=containerd.service prepare-crictl.service
+${var.use_deprecated_docker_runtime ? "Requires=docker.service" : "Requires=containerd.service prepare-crictl.service" }
 After=network-online.target
 [Service]
 Type=oneshot
@@ -21,9 +27,9 @@ ExecStart=/opt/bin/cfssl-new-scheduler-cert
 ExecStart=/opt/bin/cfssl-new-controller-manager-cert
 # Hack to reload certs on control plane tier
 #  https://github.com/kubernetes/kubernetes/issues/46287
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-controller-manager)"
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-apiserver)"
-ExecStart=-/bin/sh -c "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-scheduler)"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-controller-manager" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-controller-manager)"}"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-apiserver)" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-apiserver)"}"
+ExecStart=-/bin/sh -c "${var.use_deprecated_docker_runtime ? "docker restart $(docker ps -q -f name=k8s_kube-scheduler)" : "/opt/bin/crictl stop $(/opt/bin/crictl ps -q --label io.kubernetes.container.name=kube-scheduler)"}"
 ExecStart=/usr/bin/systemctl try-restart kubelet.service
 Restart=on-failure
 RestartSec=10

modules/cert-refresh-node/main.tf

@@ -2,13 +2,19 @@ variable "on_calendar" {
   type = string
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 data "ignition_systemd_unit" "cert-refresh" {
   name = "cert-refresh.service"
 
   content = <<EOS
 [Unit]
 Description=Fetch new certificates from cfssl server and restart components to reload certs
-Requires=containerd.service
+${var.use_deprecated_docker_runtime ? "Requires=docker.service" : "Requires=containerd.service" }
 After=network-online.target
 [Service]
 Type=oneshot

node-common.tf

@@ -84,6 +84,7 @@ data "template_file" "node-kubelet-conf" {
     kubelet_cgroup_v2_runtime_enabled = var.kubelet_cgroup_v2_runtime_enabled
     system_reserved_cpu               = var.system_reserved_cpu
     system_reserved_memory            = var.system_reserved_memory
+    use_deprecated_docker_runtime     = var.use_deprecated_docker_runtime
   }
 }
 
@@ -138,6 +139,7 @@ data "ignition_file" "prometheus-ro-rootfs" {
 }
 
 module "cert-refresh-node" {
-  source      = "./modules/cert-refresh-node"
-  on_calendar = var.cfssl_node_renew_timer
+  source                        = "./modules/cert-refresh-node"
+  on_calendar                   = var.cfssl_node_renew_timer
+  use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
 }

resources/docker-dropin.conf

@@ -1,2 +1,6 @@
 [Service]
+%{ if use_deprecated_docker_runtime ~}
+Environment=DOCKER_OPTS="--log-opt max-size=100m --log-opt max-file=1"
+%{ else ~}
 Environment=DOCKER_OPTS="--containerd=/run/containerd/containerd.sock --log-opt max-size=100m --log-opt max-file=1"
+%{ endif ~}

resources/master-kubelet-conf.yaml

@@ -10,7 +10,7 @@ authentication:
     clientCAFile: "/etc/kubernetes/ssl/ca.pem"
 authorization:
   mode: AlwaysAllow
-%{ if kubelet_cgroup_v2_runtime_enabled }
+%{ if (kubelet_cgroup_v2_runtime_enabled || use_deprecated_docker_runtime) }
 cgroupDriver: systemd
 %{ endif ~}
 clusterDNS:${cluster_dns}

resources/master-kubelet.service

@@ -1,7 +1,12 @@
 [Unit]
 Description=Kubernetes Kubelet
+%{ if use_deprecated_docker_runtime ~}
+Requires=docker.service
+After=docker.service
+%{ else ~}
 Requires=containerd.service
 After=containerd.service
+%{ endif ~}
 [Service]
 EnvironmentFile=-/etc/kubernetes/config/kubeletenv
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/config
@@ -21,8 +26,12 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 #  Flag --network-plugin has been deprecated, will be removed along with dockershim.
 ExecStart=${kubelet_binary_path} \
   --config=/etc/kubernetes/config/master-kubelet-conf.yaml \
+%{ if use_deprecated_docker_runtime ~}
+  --container-runtime=docker \
+%{ else ~}
   --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
   --container-runtime=remote \
+%{ endif ~}
   --exit-on-lock-contention \
   --hostname-override="$${NODE_HOSTNAME}" \
   --kubeconfig=/var/lib/kubelet/kubeconfig \

resources/node-kubelet-conf.yaml

@@ -10,7 +10,7 @@ authentication:
     clientCAFile: "/etc/kubernetes/ssl/ca.pem"
 authorization:
   mode: AlwaysAllow
-%{ if kubelet_cgroup_v2_runtime_enabled }
+%{ if (kubelet_cgroup_v2_runtime_enabled || use_deprecated_docker_runtime)}
 cgroupDriver: systemd
 %{ endif ~}
 clusterDNS:${cluster_dns}

resources/node-kubelet.service

@@ -1,7 +1,12 @@
 [Unit]
 Description=Kubernetes Kubelet
+%{ if use_deprecated_docker_runtime ~}
+Requires=docker.service
+After=docker.service
+%{ else ~}
 Requires=containerd.service
 After=containerd.service
+%{ endif ~}
 [Service]
 EnvironmentFile=-/etc/kubernetes/config/kubeletenv
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/config
@@ -17,8 +22,12 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 # args below `--v=0` are deprecated
 ExecStart=${kubelet_binary_path} \
   --config=/etc/kubernetes/config/node-kubelet-conf.yaml \
+%{ if use_deprecated_docker_runtime ~}
+  --container-runtime=docker \
+%{ else ~}
   --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
   --container-runtime=remote \
+%{ endif ~}
   --exit-on-lock-contention \
   --hostname-override="$${NODE_HOSTNAME}" \
   --kubeconfig=/var/lib/kubelet/kubeconfig \

variables.tf

@@ -265,6 +265,12 @@ variable "containerd_no_shim" {
   type        = bool
 }
 
+variable "use_deprecated_docker_runtime" {
+  description = "Use legacy docker container runtime"
+  default     = false
+  type        = bool
+}
+
 locals {
   # Comma separated list for cli flas use, example output:
   # `ExpandPersistentVolumes=true,PodShareProcessNamespace=true,AdvancedAuditing=false`

worker.tf

@@ -7,11 +7,12 @@ data "template_file" "worker-kubelet" {
   template = file("${path.module}/resources/node-kubelet.service")
 
   vars = {
-    kubelet_binary_path = "/opt/bin/kubelet"
-    cloud_provider      = var.cloud_provider
-    get_hostname        = var.node_name_command[var.cloud_provider]
-    labels              = "role=worker"
-    taints              = ""
+    kubelet_binary_path           = "/opt/bin/kubelet"
+    cloud_provider                = var.cloud_provider
+    get_hostname                  = var.node_name_command[var.cloud_provider]
+    labels                        = "role=worker"
+    taints                        = ""
+    use_deprecated_docker_runtime = var.use_deprecated_docker_runtime
   }
 }