I try to expose dashboard with aws alb controller but no luck

[ahoehma]

I try to use terraform to rollout dashboard with helm and my own alb ingress:

resource "helm_release" "kubernetes-dashboard" {
  name       = "kubernetes-dashboard"
  repository = "https://kubernetes.github.io/dashboard"
  chart      = "kubernetes-dashboard"
  namespace  = "kubernetes-dashboard"
  version    = "7.0.0-alpha1"
  values = [
    <<-YAML
cert-manager:
  enabled: false 
nginx:
  enabled: false
metrics-server:
  enabled: false
app:
  ingress:
    enabled: false 
YAML
    ]
}

resource "kubernetes_ingress_v1" "kubernetes-dashboard-ingress" {
  depends_on = [ helm_release.kubernetes-dashboard ]
  wait_for_load_balancer = true
  metadata {
    name      = "kubernetes-dashboard-ingress"
    namespace = "kubernetes-dashboard"
    annotations = {
      "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":443}]"
      "alb.ingress.kubernetes.io/scheme"= "internal"
      "alb.ingress.kubernetes.io/success-codes"= "200"
      "alb.ingress.kubernetes.io/target-type"= "ip"
    }
  }
  spec {
    ingress_class_name = "alb"
    rule {
      host = "kubernetes-dashboard.acme.cloud"
      http {
        # order is important! prio 1 is matching /api, prio 2 is matching /
        path {
          path = "/api"
          path_type = "Prefix"
          backend {
            service {
              name = "kubernetes-dashboard-api"
              port {
                name = "api"
              }
            }
          }
        }
        path {
          path = "/"
          path_type = "Prefix"
          backend {
            service {
              name = "kubernetes-dashboard-web"
              port {
                name = "web"
              }
            }
          }
        }
      }
    }
  }
}

output "load_balancer_hostname" {
  value = kubernetes_ingress_v1.kubernetes-dashboard-ingress.status.0.load_balancer.0.ingress.0.hostname
}

But I don't get these new split between "api" and "web" running :-(

I see that k8s is spinning up a aws alb ... 2 targets are registered

/api/* -> group-1

/* -> group-2

But group-1 is always unhealthy : Health checks failed with these codes: [404]

(I guess because of the wrong routing of /api into "service/kubernetes-dashboard-api" ... may this service is listen on "/" only?!

Any hint how to define the ingress spec?

[ahoehma]

I found a way ... but its little bit ugly :)

################################################################################
# Kubernetes dashboard 
# https://github.com/kubernetes/dashboard
################################################################################

#
# Commands to know in case of problems:
#
# check aws elb controller logs     : k logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller -f
#
# get dashboard resouces            : k get pods,svc,ingress -n kubernetes-dashboard -o wide
# delete dashboard ingress manually : k delete ingress kubernetes-dashboard-ingress -n kubernetes-dashboard
#
# get ingress                       : k get ingress kubernetes-dashboard-ingress -n kubernetes-dashboard -o yaml
# describe ingress                  : k describe ingress kubernetes-dashboard-ingress -n kubernetes-dashboard
#
# proxy                             : k proxy + open http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard-web:8000/proxy/
#

#
# Some constants
#

locals {
  # name of the created aws alb
  kubernetes_dashboard_alb_name = "my-dashboard-alb" # max 32 chars!!!
  # this hostname is used to create route53 record entry
  kubernetes_dashboard_host = "kubernetes-dashboard.xxxxxxxxxxx"
  # k8s namespace
  kubernetes_dashboard_namespace = "kubernetes-dashboard"
  # chart version
  kubernetes_dashboard_version = "7.0.0-alpha1"
}

#
# Normally all the following code must not be changed by the user
#

# we disable chart's cert-manager and metrics-server because we have this already in our cluster
# we disable chart's nginx-ingress because we want to use aws elb ingress
# we disable chart's ingress because we want to use aws elb ingress

resource "helm_release" "kubernetes-dashboard" {
  name       = "kubernetes-dashboard"
  repository = "https://kubernetes.github.io/dashboard"
  chart      = "kubernetes-dashboard"
  namespace  = local.kubernetes_dashboard_namespace
  version    = local.kubernetes_dashboard_version
  values = [
    <<-YAML
cert-manager:
  enabled: false 
metrics-server:
  enabled: false
nginx:
  enabled: false
app:
  ingress:
    enabled: false 
YAML
    ]
}

# the helm chart for kubernetes-dashboard supports only "nginx-ingress" out of the box but we want to use aws elb ingress
# so lets put this aws elb ingress on top of the helm chart
# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/ingress_v1
resource "kubernetes_ingress_v1" "kubernetes-dashboard-ingress" {
  depends_on = [ helm_release.kubernetes-dashboard ]
  wait_for_load_balancer = true
  metadata {
    name      = "kubernetes-dashboard-ingress"
    namespace = local.kubernetes_dashboard_namespace
    annotations = {
      # https://github.com/kubernetes-sigs/aws-load-balancer-controller/tree/v2.6.2
      # https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/
      "alb.ingress.kubernetes.io/load-balancer-name" = local.kubernetes_dashboard_alb_name
      "alb.ingress.kubernetes.io/healthcheck-protocol"= "HTTP"
      "alb.ingress.kubernetes.io/healthcheck-interval-seconds"= "15"
      "alb.ingress.kubernetes.io/healthcheck-port" = "traffic-port"
      "alb.ingress.kubernetes.io/healthcheck-timeout-seconds"= "4"
      "alb.ingress.kubernetes.io/healthy-threshold-count"= "2"
      "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":443}]"
      "alb.ingress.kubernetes.io/scheme"= "internal"
      "alb.ingress.kubernetes.io/target-type"= "ip"
      "alb.ingress.kubernetes.io/backend-protocol" = "HTTP"
      "alb.ingress.kubernetes.io/success-codes" = "200-499" # the api-service return 404 on "/" healthcheck so we have to include this http status here to get a healthy target group
    }
  }
  spec {
    ingress_class_name = "alb"
    rule {
      # this hostname is used to create route53 record entry
      host = local.kubernetes_dashboard_host
      http {
        # order is important! prio 1 is matching /api, prio 2 is matching /
        path {
          path = "/api/"
          path_type = "Prefix"
          backend {
            service {
              name = "kubernetes-dashboard-api"
              port {
                name = "api"
              }
            }
          }
        }
        path {
          path = "/"
          path_type = "Prefix"
          backend {
            service {
              name = "kubernetes-dashboard-web"
              port {
                name = "web"
              }
            }
          }
        }
      }
    }
  }
}

#
# Outputs
#

data "aws_alb" "kubernetes_dashboard_alb_details" {
  name = local.kubernetes_dashboard_alb_name
}

output "kubernetes_dashboard_alb_dns_name" {
  value = data.aws_alb.kubernetes_dashboard_alb_details.dns_name
}

output "kubernetes_dashboard_lb_arn" {
  value = data.aws_alb.kubernetes_dashboard_alb_details.id
}

output "kubernetes_dashboard_alb_hostname" {
  value = kubernetes_ingress_v1.kubernetes-dashboard-ingress.status.0.load_balancer.0.ingress.0.hostname
}