Merge pull request hashicorp#685 from hashicorp/nightly-tfe-tests

Nightly tfe tests workflow

.github/scripts/build_instance/main.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"time"
 
 	tfe "github.com/hashicorp/go-tfe"
 )
@@ -39,9 +40,19 @@ func main() {
 		log.Fatalf("client initialization error: %v", err)
 	}
 
-	if _, err = createRun(ctx, client); err != nil {
+	var runID string
+	if runID, err = createRun(ctx, client); err != nil {
 		log.Fatal(err)
 	}
+
+	// we should only wait if we are creating an instance
+	if !isDestroy {
+		if err = waitForRun(ctx, client, runID); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Printf("Run with ID successfully applied: %s", runID)
+	}
 }
 
 func createRun(ctx context.Context, client *tfe.Client) (string, error) {
@@ -65,7 +76,38 @@ func createRun(ctx context.Context, client *tfe.Client) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to trigger run: %w", err)
 	}
+	log.Printf("Run created: %s", run.ID)
 
-	log.Printf("Created run: %s\n", run.ID)
 	return run.ID, nil
 }
+
+func waitForRun(ctx context.Context, client *tfe.Client, runID string) error {
+	// The run should take about 5 minutes to complete;
+	// polling the status of the run every 20 seconds or so
+	// should be frequent enough. It's also long enough to ensure
+	// no ticks are dropped.
+	ticker := time.NewTicker(time.Second * 20)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("Context canceled: %w", ctx.Err())
+		case <-ticker.C:
+			run, err := client.Runs.Read(ctx, runID)
+			if err != nil {
+				return err
+			}
+
+			switch run.Status {
+			case tfe.RunCanceled, tfe.RunErrored, tfe.RunDiscarded:
+				return fmt.Errorf("Could not complete run: %s", string(run.Status))
+			case tfe.RunApplied:
+				// run is complete
+				return nil
+			default:
+				log.Printf("Polling run %s, has status: %s", runID, string(run.Status))
+			}
+		}
+	}
+}

.github/workflows/nightly-tfe-test.yml

@@ -0,0 +1,106 @@
+name: Nightly TFE Tests
+on:
+  schedule:
+    - cron: 0 0 * * *
+
+jobs:
+  instance:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Sync dependencies
+        run: |
+          go mod download
+          go mod tidy
+      - name: Build nightly TFE instance
+        env:
+          TFE_TOKEN: ${{ secrets.TF_WORKFLOW_TFLOCAL_CLOUD_TFC_TOKEN }}
+        run: |
+          cd .github/scripts/build_instance
+          go run . -w tflocal-terraform-provider-tfe-nightly
+
+  tests:
+    name: run
+    needs: instance
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        parallel: [3]
+        index: [0, 1, 2]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Sync dependencies
+        run: |
+          go get -v -t -d ./...
+          go mod tidy
+
+      - name: Install gotestsum
+        run: go install gotest.tools/gotestsum@latest
+
+      - name: Split acceptance tests
+        id: test_split
+        uses: brandonc/go-test-split-action@v1
+        with:
+          index: ${{ matrix.index }}
+          total: ${{ matrix.parallel }}
+
+      - name: Fetch Outputs
+        env:
+          TFE_TOKEN: "${{ secrets.TF_WORKFLOW_TFLOCAL_CLOUD_TFC_TOKEN }}"
+        run: |
+          go run .github/scripts/fetch_outputs/main.go hashicorp-v2 tflocal-terraform-provider-tfe-nightly
+
+      - name: Run Tests
+        env:
+          ENABLE_TFE: "1"
+          TFC_RUN_TASK_URL: "https://httpstat.us/200"
+          GITHUB_POLICY_SET_IDENTIFIER: "hashicorp/test-policy-set"
+          GITHUB_REGISTRY_MODULE_IDENTIFIER: "hashicorp/terraform-random-module"
+          GITHUB_WORKSPACE_IDENTIFIER: "hashicorp/terraform-random-1"
+          GITHUB_WORKSPACE_BRANCH: "test"
+          GITHUB_TOKEN: "${{ secrets.TESTING_GITHUB_TOKEN }}"
+          MOD_PROVIDER: github.com/hashicorp/terraform-provider-tfe
+          MOD_TFE: github.com/hashicorp/terraform-provider-tfe/tfe
+          MOD_VERSION: github.com/hashicorp/terraform-provider-tfe/version
+        run: |
+          source $HOME/.env
+          gotestsum --format short-verbose -- $MOD_PROVIDER $MOD_TFE $MOD_VERSION -timeout=30m -run "${{ steps.test_split.outputs.run }}"
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: tests
+    if: "${{ always() }}"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+
+      - name: Destroy nightly TFE instance
+        env:
+          TFE_TOKEN: ${{ secrets.TF_WORKFLOW_TFLOCAL_CLOUD_TFC_TOKEN }}
+        run: |
+          cd .github/scripts/build_instance
+          go run . -w tflocal-terraform-provider-tfe-nightly -d

.github/workflows/test.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Install gotestsum
         run: go install gotest.tools/gotestsum@latest
 
-      - name: Split integration tests
+      - name: Split acceptance tests
         id: test_split
         uses: brandonc/go-test-split-action@v1
         with: