Merge pull request #36208 from bprashanth/curl_timeout

Automatic merge from submit-queue Stricter timeouts for nodePort curling If the timeouts are indeed because of #34665 (comment), stricter timeouts will probably surface as a more isolated failure
kubernetes · Nov 4, 2016 · 3adc580 · 3adc580
2 parents a96f028 + 0242997
commit 3adc580
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 9 deletions.
diff --git a/test/e2e/density.go b/test/e2e/density.go
@@ -313,7 +313,7 @@ var _ = framework.KubeDescribe("Density", func() {
 		// In large clusters we may get to this point but still have a bunch
 		// of nodes without Routes created. Since this would make a node
 		// unschedulable, we need to wait until all of them are schedulable.
-		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c))
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.NodeSchedulableTimeout))
 		masters, nodes = framework.GetMasterAndWorkerNodesOrDie(c)
 		nodeCount = len(nodes.Items)
 		Expect(nodeCount).NotTo(BeZero())

diff --git a/test/e2e/empty.go b/test/e2e/empty.go
@@ -32,7 +32,7 @@ var _ = framework.KubeDescribe("Empty [Feature:Empty]", func() {
 		ns := f.Namespace.Name
 
 		// TODO: respect --allow-notready-nodes flag in those functions.
-		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c))
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.NodeSchedulableTimeout))
 		framework.WaitForAllNodesHealthy(c, time.Minute)
 
 		err := framework.CheckTestingNSDeletedExcept(c, ns)

diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go
@@ -223,7 +223,7 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ
 		// busybox timeout doesn't support non-integer values.
 		cmd = fmt.Sprintf("echo 'hostName' | timeout -t 2 nc -w 1 -u %s %d", targetIP, targetPort)
 	} else {
-		cmd = fmt.Sprintf("curl -q -s --connect-timeout 1 http://%s:%d/hostName", targetIP, targetPort)
+		cmd = fmt.Sprintf("timeout -t 15 curl -q -s --connect-timeout 1 --max-time 10 http://%s:%d/hostName", targetIP, targetPort)
 	}
 
 	// TODO: This simply tells us that we can reach the endpoints. Check that
@@ -435,7 +435,7 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
 	config.setupCore(selector)
 
 	By("Getting node addresses")
-	ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet))
+	ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute))
 	nodeList := GetReadySchedulableNodesOrDie(config.f.ClientSet)
 	config.ExternalAddrs = NodeAddresses(nodeList, api.NodeExternalIP)
 	if len(config.ExternalAddrs) < 2 {
@@ -486,7 +486,7 @@ func shuffleNodes(nodes []api.Node) []api.Node {
 }
 
 func (config *NetworkingTestConfig) createNetProxyPods(podName string, selector map[string]string) []*api.Pod {
-	ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet))
+	ExpectNoError(WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute))
 	nodeList := GetReadySchedulableNodesOrDie(config.f.ClientSet)
 
 	// To make this test work reasonably fast in large clusters,

diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go
@@ -170,6 +170,10 @@ const (
 
 	// TODO(justinsb): Avoid hardcoding this.
 	awsMasterIP = "172.20.0.9"
+
+	// Default time to wait for nodes to become schedulable.
+	// Set so high for scale tests.
+	NodeSchedulableTimeout = 4 * time.Hour
 )
 
 var (
@@ -2416,11 +2420,11 @@ func GetReadySchedulableNodesOrDie(c clientset.Interface) (nodes *api.NodeList)
 	return nodes
 }
 
-func WaitForAllNodesSchedulable(c clientset.Interface) error {
-	Logf("Waiting up to %v for all (but %d) nodes to be schedulable", 4*time.Hour, TestContext.AllowedNotReadyNodes)
+func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) error {
+	Logf("Waiting up to %v for all (but %d) nodes to be schedulable", timeout, TestContext.AllowedNotReadyNodes)
 
 	var notSchedulable []*api.Node
-	return wait.PollImmediate(30*time.Second, 4*time.Hour, func() (bool, error) {
+	return wait.PollImmediate(30*time.Second, timeout, func() (bool, error) {
 		notSchedulable = nil
 		opts := api.ListOptions{
 			ResourceVersion: "0",

diff --git a/test/e2e/load.go b/test/e2e/load.go
@@ -101,7 +101,7 @@ var _ = framework.KubeDescribe("Load capacity", func() {
 		// In large clusters we may get to this point but still have a bunch
 		// of nodes without Routes created. Since this would make a node
 		// unschedulable, we need to wait until all of them are schedulable.
-		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(clientset))
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(clientset, framework.NodeSchedulableTimeout))
 
 		ns = f.Namespace.Name
 		nodes := framework.GetReadySchedulableNodesOrDie(clientset)