Merge pull request #384 from hashicorp/resolve-flaky-tests

Resolve flaky tests

helper_test.go

@@ -7,20 +7,25 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
 	uuid "github.com/hashicorp/go-uuid"
 )
 
 const badIdentifier = "! / nope" //nolint
+const tickDuration = 2
 
 // Memoize test account details
 var _testAccountDetails *TestAccountDetails
 
+type retryableFn func() (interface{}, error)
+
 func testClient(t *testing.T) *Client {
 	client, err := NewClient(nil)
 	client.RetryServerErrors(true) // because occasionally we get a 500 internal when deleting an organization's workspace
@@ -1161,6 +1166,85 @@ func createVariableSetVariable(t *testing.T, client *Client, vs *VariableSet, op
 	}
 }
 
+func waitForSVOutputs(t *testing.T, client *Client, svID string) {
+	t.Helper()
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	go func() {
+		_, err := retry(func() (interface{}, error) {
+			outputs, err := client.StateVersions.ListOutputs(context.Background(), svID, nil)
+			if err != nil {
+				return nil, err
+			}
+
+			if len(outputs.Items) == 0 {
+				return nil, errors.New("no state version outputs found")
+			}
+
+			return outputs, nil
+		})
+		if err != nil {
+			t.Error(err)
+		}
+
+		wg.Done()
+	}()
+
+	wg.Wait()
+}
+
+func waitForRunLock(t *testing.T, client *Client, workspaceID string) {
+	t.Helper()
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	go func() {
+		_, err := retry(func() (interface{}, error) {
+			ws, err := client.Workspaces.ReadByID(context.Background(), workspaceID)
+			if err != nil {
+				return nil, err
+			}
+
+			if !ws.Locked {
+				return nil, errors.New("workspace is not locked by run")
+			}
+
+			return ws, nil
+		})
+		if err != nil {
+			t.Error(err)
+		}
+
+		wg.Done()
+	}()
+
+	wg.Wait()
+}
+
+func retry(f retryableFn) (interface{}, error) {
+	tick := time.NewTicker(tickDuration * time.Second)
+
+	retries := 0
+	maxRetries := 5
+
+	for {
+		select {
+		case <-tick.C:
+			res, err := f()
+			if err == nil {
+				return res, err
+			}
+
+			if retries >= maxRetries {
+				return nil, err
+			}
+
+			retries += 1
+		}
+	}
+}
+
 func genSha(t *testing.T, secret, data string) string {
 	h := hmac.New(sha256.New, []byte(secret))
 	_, err := h.Write([]byte(data))

state_version_integration_test.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -313,7 +312,7 @@ func TestStateVersionsReadWithOptions(t *testing.T) {
 	defer svTestCleanup()
 
 	// give TFC some time to process the statefile and extract the outputs.
-	time.Sleep(waitForStateVersionOutputs)
+	waitForSVOutputs(t, client, svTest.ID)
 
 	t.Run("when the state version exists", func(t *testing.T) {
 		curOpts := &StateVersionReadOptions{
@@ -377,11 +376,11 @@ func TestStateVersionsCurrentWithOptions(t *testing.T) {
 	wTest1, wTest1Cleanup := createWorkspace(t, client, nil)
 	defer wTest1Cleanup()
 
-	_, svTestCleanup := createStateVersion(t, client, 0, wTest1)
+	svTest, svTestCleanup := createStateVersion(t, client, 0, wTest1)
 	defer svTestCleanup()
 
 	// give TFC some time to process the statefile and extract the outputs.
-	time.Sleep(waitForStateVersionOutputs)
+	waitForSVOutputs(t, client, svTest.ID)
 
 	t.Run("when the state version exists", func(t *testing.T) {
 		curOpts := &StateVersionCurrentOptions{
@@ -438,7 +437,7 @@ func TestStateVersionOutputs(t *testing.T) {
 	defer svTestCleanup()
 
 	// give TFC some time to process the statefile and extract the outputs.
-	time.Sleep(waitForStateVersionOutputs)
+	waitForSVOutputs(t, client, sv.ID)
 
 	t.Run("when the state version exists", func(t *testing.T) {
 		outputs, err := client.StateVersions.ListOutputs(ctx, sv.ID, nil)

state_version_output_integration_test.go

@@ -6,26 +6,23 @@ package tfe
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const waitForStateVersionOutputs = 1000 * time.Millisecond
-
 func TestStateVersionOutputsRead(t *testing.T) {
 	client := testClient(t)
 	ctx := context.Background()
 
 	wTest1, wTest1Cleanup := createWorkspace(t, client, nil)
 	defer wTest1Cleanup()
 
-	_, svTestCleanup := createStateVersion(t, client, 0, wTest1)
+	svTest, svTestCleanup := createStateVersion(t, client, 0, wTest1)
 	defer svTestCleanup()
 
 	// give TFC some time to process the statefile and extract the outputs.
-	time.Sleep(waitForStateVersionOutputs)
+	waitForSVOutputs(t, client, svTest.ID)
 
 	curOpts := &StateVersionCurrentOptions{
 		Include: []StateVersionIncludeOpt{SVoutputs},

workspace_integration_test.go

@@ -339,7 +339,7 @@ func TestWorkspacesReadWithOptions(t *testing.T) {
 	defer svTestCleanup()
 
 	// give TFC some time to process the statefile and extract the outputs.
-	time.Sleep(waitForStateVersionOutputs)
+	waitForSVOutputs(t, client, svTest.ID)
 
 	t.Run("when options to include resource", func(t *testing.T) {
 		opts := &WorkspaceReadOptions{
@@ -791,13 +791,22 @@ func TestWorkspacesUnlock(t *testing.T) {
 	wTest, wTestCleanup := createWorkspace(t, client, orgTest)
 	defer wTestCleanup()
 
+	wTest2, wTest2Cleanup := createWorkspace(t, client, orgTest)
+	defer wTest2Cleanup()
+
 	w, err := client.Workspaces.Lock(ctx, wTest.ID, WorkspaceLockOptions{})
 	if err != nil {
 		orgTestCleanup()
 	}
 	require.NoError(t, err)
 	require.True(t, w.Locked)
 
+	_, rTestCleanup := createRun(t, client, wTest2)
+	defer rTestCleanup()
+
+	// Wait for wTest2 to be locked by a run
+	waitForRunLock(t, client, wTest2.ID)
+
 	t.Run("with valid options", func(t *testing.T) {
 		w, err := client.Workspaces.Unlock(ctx, wTest.ID)
 		require.NoError(t, err)
@@ -810,13 +819,7 @@ func TestWorkspacesUnlock(t *testing.T) {
 	})
 
 	t.Run("when a workspace is locked by a run", func(t *testing.T) {
-		wTest2, wTest2Cleanup := createWorkspace(t, client, orgTest)
-		defer wTest2Cleanup()
-
-		_, rTestCleanup := createRun(t, client, wTest2)
-		defer rTestCleanup()
-
-		_, err := client.Workspaces.Unlock(ctx, wTest2.ID)
+		_, err = client.Workspaces.Unlock(ctx, wTest2.ID)
 		assert.Equal(t, ErrWorkspaceLockedByRun, err)
 	})