Merge pull request #78246 from mborsz/kubemark

Migrate kubemark to e2e-up/e2e-down scripts

cluster/kubemark/gce/config-default.sh

@@ -23,92 +23,22 @@
 
 source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 
-GCLOUD=gcloud
-ZONE=${KUBE_GCE_ZONE:-us-central1-b}
-REGION=${ZONE%-*}
-NUM_NODES=${KUBEMARK_NUM_NODES:-10}
-NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0}
-MASTER_SIZE=${KUBEMARK_MASTER_SIZE:-n1-standard-$(get-master-size)}
-MASTER_DISK_TYPE=pd-ssd
-MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-$(get-master-disk-size)}
-MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)}
-REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
-PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
-NODE_ACCELERATORS=${NODE_ACCELERATORS:-""}
-CREATE_CUSTOM_NETWORK=${CREATE_CUSTOM_NETWORK:-false}
-EVENT_PD=${EVENT_PD:-false}
-
-MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci}
-NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci}
-MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-beta-73-11647-64-0}
-MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
 CLEANUP_KUBEMARK_IMAGE=${CLEANUP_KUBEMARK_IMAGE:-true}
 
-# GPUs supported in GCE do not have compatible drivers in Debian 7.
-if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
-    NODE_ACCELERATORS=""
-fi
-
-NETWORK=${KUBE_GCE_NETWORK:-e2e}
-if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then
-  SUBNETWORK="${SUBNETWORK:-${NETWORK}-custom-subnet}"
-fi
-INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}"
-MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
-AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-kubemark-aggregator"
-MASTER_TAG="kubemark-master"
-EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
-MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
-CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-$(get-cluster-ip-range)}"
-RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
-TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
-KUBE_APISERVER_REQUEST_TIMEOUT=300
-ETCD_COMPACTION_INTERVAL_SEC="${KUBEMARK_ETCD_COMPACTION_INTERVAL_SEC:-}"
-
-# Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.1.12-1) if you need
-# non-default version.
-ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"
-ETCD_VERSION="${TEST_ETCD_VERSION:-}"
-ETCD_SERVERS="${KUBEMARK_ETCD_SERVERS:-}"
-ETCD_SERVERS_OVERRIDES="${KUBEMARK_ETCD_SERVERS_OVERRIDES:-}"
-
-# Storage backend. 'etcd2' and 'etcd3' are supported.
-STORAGE_BACKEND=${STORAGE_BACKEND:-}
-# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported.
-STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-}
-
-# Default Log level for all components in test clusters and variables to override it in specific components.
-TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=4}"
-API_SERVER_TEST_LOG_LEVEL="${API_SERVER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
-CONTROLLER_MANAGER_TEST_LOG_LEVEL="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
-SCHEDULER_TEST_LOG_LEVEL="${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+# NUM_NODES is used by start-kubemark.sh to determine a correct number of replicas.
+NUM_NODES=${KUBEMARK_NUM_NODES:-10}	
+NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0}
 
 HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
 HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
 
-TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=16}"
-TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}"
-TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:-}"
-
 # ContentType used by all components to communicate with apiserver.
 TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
 
-KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
-
-CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota}"
-
-# Master components' test arguments.
-APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
-CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
-SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
-
 # Hollow-node components' test arguments.
 HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}"
 HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}"
 
-SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
-ALLOCATE_NODE_CIDRS=true
-
 # Optional: Enable cluster autoscaler.
 ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}"
 # When using Cluster Autoscaler, always start with one hollow-node replica.
@@ -126,12 +56,4 @@ fi
 ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}"
 KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}"
 
-# Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
-
-# Enable a simple "AdvancedAuditing" setup for testing.
-ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-false}"
-
-# The number of services that are allowed to sync concurrently. Will be passed
-# into kube-controller-manager via `--concurrent-service-syncs`
-CONCURRENT_SERVICE_SYNCS="${CONCURRENT_SERVICE_SYNCS:-}"
+CLEANUP_KUBEMARK_IMAGE=false

test/kubemark/gce/util.sh

@@ -31,127 +31,65 @@ function authenticate-docker {
   gcloud beta auth configure-docker -q
 }
 
-# This function isn't too robust to race, but that should be ok given its one-off usage during setup.
-function get-or-create-master-ip {
-  MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
-    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') 2>/dev/null || true
-
-  if [[ -z "${MASTER_IP:-}" ]]; then
-    run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
-    --project "${PROJECT}" \
-    --region "${REGION}" -q
-
-    MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
-    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
-  fi
-}
-
-function create-master-instance-with-resources {
-  GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}")
-  # Override the master image project to cos-cloud for COS images staring with `cos` string prefix.
-  DEFAULT_GCI_PROJECT=google-containers
-  if [[ "${GCI_VERSION}" == "cos"* ]]; then
-      DEFAULT_GCI_PROJECT=cos-cloud
-  fi
-  MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}}
-
-  run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
-    "${GCLOUD_COMMON_ARGS[@]}" \
-    --type "${MASTER_DISK_TYPE}" \
-    --size "${MASTER_DISK_SIZE}" &
-
-  if [ "${EVENT_PD:-}" == "true" ]; then
-    run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
-      "${GCLOUD_COMMON_ARGS[@]}" \
-      --type "${MASTER_DISK_TYPE}" \
-      --size "${MASTER_DISK_SIZE}" &
-  fi
-
-  get-or-create-master-ip &
-
-  wait
-
-  run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \
-    "${GCLOUD_COMMON_ARGS[@]}" \
-    --address "${MASTER_IP}" \
-    --machine-type "${MASTER_SIZE}" \
-    --image-project="${MASTER_IMAGE_PROJECT}" \
-    --image "${MASTER_IMAGE}" \
-    --tags "${MASTER_TAG}" \
-    --subnet "${SUBNETWORK:-${NETWORK}}" \
-    --scopes "storage-ro,logging-write" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \
-    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
-
-  run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \
-    "${GCLOUD_COMMON_ARGS[@]}" \
-    --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" &
-
-  if [ "${EVENT_PD:-}" == "true" ]; then
-    echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}"
-    run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \
-   "${GCLOUD_COMMON_ARGS[@]}" \
-    --disk "${MASTER_NAME}-event-pd" \
-    --device-name="master-event-pd" &
-  fi
-
-  run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \
-    --project "${PROJECT}" \
-    --network "${NETWORK}" \
-    --source-ranges "0.0.0.0/0" \
-    --target-tags "${MASTER_TAG}" \
-    --allow "tcp:443" &
-
-  run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-internal" \
-    --project "${PROJECT}" \
-    --network "${NETWORK}" \
-    --source-ranges "10.0.0.0/8" \
-    --target-tags "${MASTER_TAG}" \
-    --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" &
-
-  wait
-}
-
-# Command to be executed is '$1'.
-# No. of retries is '$2' (if provided) or 1 (default).
-function execute-cmd-on-master-with-retries() {
-  RETRIES="${2:-1}" run-gcloud-compute-with-retries ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" --command="$1"
+function create-kubemark-master {
+  # We intentionally override env vars in subshell to preserve original values.
+  # shellcheck disable=SC2030,SC2031
+  (
+    export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
+    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
+    export KUBE_CREATE_NODES=false
+    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
+
+    # Even if the "real cluster" is private, we shouldn't manage cloud nat.
+    export KUBE_GCE_PRIVATE_CLUSTER=false
+
+    # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating
+    # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager.
+    export KUBE_GCE_ENABLE_IP_ALIASES=true
+    export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator
+
+    # Disable all addons. They are running outside of the kubemark cluster.
+    export KUBE_ENABLE_CLUSTER_AUTOSCALER=false
+    export KUBE_ENABLE_CLUSTER_DNS=false
+    export KUBE_ENABLE_NODE_LOGGING=false
+    export KUBE_ENABLE_METRICS_SERVER=false
+    export KUBE_ENABLE_CLUSTER_MONITORING="none"
+    export KUBE_ENABLE_L7_LOADBALANCING="none"
+
+    # Unset env variables set by kubetest for 'root cluster'. We need recompute them 
+    # for kubemark master.
+    # TODO(mborsz): Figure out some better way to filter out such env variables than
+    # listing them here.
+    unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE
+
+    # Set kubemark-specific overrides:
+    # for each defined env KUBEMARK_X=Y call export X=Y.
+    for var in ${!KUBEMARK_*}; do
+      dst_var=${var#KUBEMARK_}
+      val=${!var}
+      echo "Setting ${dst_var} to '${val}'"
+      export "${dst_var}"="${val}"
+    done
+
+    # Append kubemark-specific "MASTER_COMPONENTS_QPS_LIMITS".
+    # TODO(mborsz): Migrate all users of KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS to {SCHEDULER,CONTROLLER_MANAGER}_TEST_ARGS.
+    export CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
+    export SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
+
+    "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"
+    )
 }
 
-function copy-files() {
-	run-gcloud-compute-with-retries scp --recurse --zone="${ZONE}" --project="${PROJECT}" "$@"
-}
-
-function delete-master-instance-and-resources {
-  GCLOUD_COMMON_ARGS=(--project "${PROJECT}" --zone "${ZONE}" --quiet)
-
-  gcloud compute instances delete "${MASTER_NAME}" \
-      "${GCLOUD_COMMON_ARGS[@]}" || true
-
-  gcloud compute disks delete "${MASTER_NAME}-pd" \
-      "${GCLOUD_COMMON_ARGS[@]}" || true
-
-  gcloud compute disks delete "${MASTER_NAME}-event-pd" \
-      "${GCLOUD_COMMON_ARGS[@]}" &> /dev/null || true
-
-  gcloud compute addresses delete "${MASTER_NAME}-ip" \
-      --project "${PROJECT}" \
-      --region "${REGION}" \
-      --quiet || true
-
-  gcloud compute firewall-rules delete "${MASTER_NAME}-https" \
-	  --project "${PROJECT}" \
-	  --quiet || true
-
-  gcloud compute firewall-rules delete "${MASTER_NAME}-internal" \
-	  --project "${PROJECT}" \
-	  --quiet || true
-
-  if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
-	  gcloud compute instances delete "${EVENT_STORE_NAME}" \
-    	  "${GCLOUD_COMMON_ARGS[@]}" || true
-
-	  gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \
-        "${GCLOUD_COMMON_ARGS[@]}" || true
-  fi
+function delete-kubemark-master {
+  # We intentionally override env vars in subshell to preserve original values.
+  # shellcheck disable=SC2030,SC2031
+  ( 
+    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
+    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
+
+    export KUBE_DELETE_NETWORK=false
+    # Even if the "real cluster" is private, we shouldn't manage cloud nat.
+    export KUBE_GCE_PRIVATE_CLUSTER=false
+    "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"
+  )
 }