Adding RefreshState test step (#1070)

* Adding RefreshState test step (#1069)

* Adding CHANGELOG entry (#1069)

* Running TestCheckFunc with refreshed state (#1069)

* Adding validation to check conditions for RefreshState (#1069)

* Expanding comments on RefreshState (#1069)

* Removing option to override config during refresh testing, removing re-init and adding plan following refresh (#1069)

* Adding test coverage for to verify expect non-empty plan following refresh (#1069)

* Apply suggestions from code review

Co-authored-by: Brian Flad <bflad417@gmail.com>

* Adding validation to verify that refresh state is not present with config or destroy in a test step (#1069)

* Reset time during test step so that ReadContext does not mutate state and result in a diff (#1069)

* Updating website docs (#1069)

* Apply suggestions from code review

Co-authored-by: Brian Flad <bflad417@gmail.com>

* Test to verify that setting config and refresh state together is not valid (#1069)

Co-authored-by: Brian Flad <bflad417@gmail.com>

.changelog/1070.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+helper/resource: Added `TestStep` type `RefreshState` field, which enables a step that refreshes state without an explicit apply or configuration changes
+```

helper/resource/testing.go

@@ -570,6 +570,22 @@ type TestStep struct {
 	// at the end of the test step that is verifying import behavior.
 	ImportStatePersist bool
 
+	//---------------------------------------------------------------
+	// RefreshState testing
+	//---------------------------------------------------------------
+
+	// RefreshState, if true, will test the functionality of `terraform
+	// refresh` by refreshing the state, running any checks against the
+	// refreshed state, and running a plan to verify against unexpected plan
+	// differences.
+	//
+	// If the refresh is expected to result in a non-empty plan
+	// ExpectNonEmptyPlan should be set to true in the same TestStep.
+	//
+	// RefreshState cannot be the first TestStep and, it is mutually exclusive
+	// with ImportState.
+	RefreshState bool
+
 	// ProviderFactories can be specified for the providers that are valid for
 	// this TestStep. When providers are specified at the TestStep level, all
 	// TestStep within a TestCase must declare providers.

helper/resource/testing_new.go

@@ -114,7 +114,7 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 
 	logging.HelperResourceDebug(ctx, "Starting TestSteps")
 
-	// use this to track last step succesfully applied
+	// use this to track last step successfully applied
 	// acts as default for import tests
 	var appliedCfg string
 
@@ -241,6 +241,45 @@ func runNewTest(ctx context.Context, t testing.T, c TestCase, helper *plugintest
 			continue
 		}
 
+		if step.RefreshState {
+			logging.HelperResourceTrace(ctx, "TestStep is RefreshState mode")
+
+			err := testStepNewRefreshState(ctx, t, wd, step, providers)
+			if step.ExpectError != nil {
+				logging.HelperResourceDebug(ctx, "Checking TestStep ExpectError")
+				if err == nil {
+					logging.HelperResourceError(ctx,
+						"Error running refresh: expected an error but got none",
+					)
+					t.Fatalf("Step %d/%d error running refresh: expected an error but got none", stepNumber, len(c.Steps))
+				}
+				if !step.ExpectError.MatchString(err.Error()) {
+					logging.HelperResourceError(ctx,
+						fmt.Sprintf("Error running refresh: expected an error with pattern (%s)", step.ExpectError.String()),
+						map[string]interface{}{logging.KeyError: err},
+					)
+					t.Fatalf("Step %d/%d error running refresh, expected an error with pattern (%s), no match on: %s", stepNumber, len(c.Steps), step.ExpectError.String(), err)
+				}
+			} else {
+				if err != nil && c.ErrorCheck != nil {
+					logging.HelperResourceDebug(ctx, "Calling TestCase ErrorCheck")
+					err = c.ErrorCheck(err)
+					logging.HelperResourceDebug(ctx, "Called TestCase ErrorCheck")
+				}
+				if err != nil {
+					logging.HelperResourceError(ctx,
+						"Error running refresh",
+						map[string]interface{}{logging.KeyError: err},
+					)
+					t.Fatalf("Step %d/%d error running refresh: %s", stepNumber, len(c.Steps), err)
+				}
+			}
+
+			logging.HelperResourceDebug(ctx, "Finished TestStep")
+
+			continue
+		}
+
 		if step.Config != "" {
 			logging.HelperResourceTrace(ctx, "TestStep is Config mode")
 

helper/resource/testing_new_refresh_state.go

@@ -0,0 +1,97 @@
+package resource
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/davecgh/go-spew/spew"
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/internal/logging"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/internal/plugintest"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
+)
+
+func testStepNewRefreshState(ctx context.Context, t testing.T, wd *plugintest.WorkingDir, step TestStep, providers *providerFactories) error {
+	t.Helper()
+
+	spewConf := spew.NewDefaultConfig()
+	spewConf.SortKeys = true
+
+	var err error
+	// Explicitly ensure prior state exists before refresh.
+	err = runProviderCommand(ctx, t, func() error {
+		_, err = getState(ctx, t, wd)
+		if err != nil {
+			return err
+		}
+		return nil
+	}, wd, providers)
+	if err != nil {
+		t.Fatalf("Error getting state: %s", err)
+	}
+
+	err = runProviderCommand(ctx, t, func() error {
+		return wd.Refresh(ctx)
+	}, wd, providers)
+	if err != nil {
+		return err
+	}
+
+	var refreshState *terraform.State
+	err = runProviderCommand(ctx, t, func() error {
+		refreshState, err = getState(ctx, t, wd)
+		if err != nil {
+			return err
+		}
+		return nil
+	}, wd, providers)
+	if err != nil {
+		t.Fatalf("Error getting state: %s", err)
+	}
+
+	// Go through the refreshed state and verify
+	if step.Check != nil {
+		logging.HelperResourceDebug(ctx, "Calling TestStep Check for RefreshState")
+
+		if err := step.Check(refreshState); err != nil {
+			t.Fatal(err)
+		}
+
+		logging.HelperResourceDebug(ctx, "Called TestStep Check for RefreshState")
+	}
+
+	// do a plan
+	err = runProviderCommand(ctx, t, func() error {
+		return wd.CreatePlan(ctx)
+	}, wd, providers)
+	if err != nil {
+		return fmt.Errorf("Error running post-apply plan: %w", err)
+	}
+
+	var plan *tfjson.Plan
+	err = runProviderCommand(ctx, t, func() error {
+		var err error
+		plan, err = wd.SavedPlan(ctx)
+		return err
+	}, wd, providers)
+	if err != nil {
+		return fmt.Errorf("Error retrieving post-apply plan: %w", err)
+	}
+
+	if !planIsEmpty(plan) && !step.ExpectNonEmptyPlan {
+		var stdout string
+		err = runProviderCommand(ctx, t, func() error {
+			var err error
+			stdout, err = wd.SavedPlanRawStdout(ctx)
+			return err
+		}, wd, providers)
+		if err != nil {
+			return fmt.Errorf("Error retrieving formatted plan output: %w", err)
+		}
+		return fmt.Errorf("After refreshing state during this test step, a followup plan was not empty.\nstdout:\n\n%s", stdout)
+	}
+
+	return nil
+}

helper/resource/teststep_providers_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 )
@@ -1574,6 +1575,157 @@ func TestTest_TestStep_ProviderFactories_Import_External_WithoutPersistNonMatch(
 	})
 }
 
+func TestTest_TestStep_ProviderFactories_Refresh_Inline(t *testing.T) {
+	t.Parallel()
+
+	Test(t, TestCase{
+		ProviderFactories: map[string]func() (*schema.Provider, error){
+			"random": func() (*schema.Provider, error) { //nolint:unparam // required signature
+				return &schema.Provider{
+					ResourcesMap: map[string]*schema.Resource{
+						"random_password": {
+							CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics {
+								d.SetId("id")
+								err := d.Set("min_special", 10)
+								if err != nil {
+									panic(err)
+								}
+								return nil
+							},
+							DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics {
+								return nil
+							},
+							ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics {
+								err := d.Set("min_special", 2)
+								if err != nil {
+									panic(err)
+								}
+								return nil
+							},
+							Schema: map[string]*schema.Schema{
+								"min_special": {
+									Computed: true,
+									Type:     schema.TypeInt,
+								},
+
+								"id": {
+									Computed: true,
+									Type:     schema.TypeString,
+								},
+							},
+						},
+					},
+				}, nil
+			},
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "random_password" "test" { }`,
+				Check:  TestCheckResourceAttr("random_password.test", "min_special", "10"),
+			},
+			{
+				RefreshState: true,
+				Check:        TestCheckResourceAttr("random_password.test", "min_special", "2"),
+			},
+			{
+				Config: `resource "random_password" "test" { }`,
+				Check:  TestCheckResourceAttr("random_password.test", "min_special", "2"),
+			},
+		},
+	})
+}
+
+func TestTest_TestStep_ProviderFactories_RefreshWithPlanModifier_Inline(t *testing.T) {
+	t.Parallel()
+
+	Test(t, TestCase{
+		ProviderFactories: map[string]func() (*schema.Provider, error){
+			"random": func() (*schema.Provider, error) { //nolint:unparam // required signature
+				return &schema.Provider{
+					ResourcesMap: map[string]*schema.Resource{
+						"random_password": {
+							CustomizeDiff: customdiff.All(
+								func(ctx context.Context, d *schema.ResourceDiff, meta interface{}) error {
+									special := d.Get("special").(bool)
+									if special == true {
+										err := d.SetNew("special", false)
+										if err != nil {
+											panic(err)
+										}
+									}
+									return nil
+								},
+							),
+							CreateContext: func(ctx context.Context, d *schema.ResourceData, i interface{}) diag.Diagnostics {
+								d.SetId("id")
+								err := d.Set("special", false)
+								if err != nil {
+									panic(err)
+								}
+								return nil
+							},
+							DeleteContext: func(_ context.Context, _ *schema.ResourceData, _ interface{}) diag.Diagnostics {
+								return nil
+							},
+							ReadContext: func(_ context.Context, d *schema.ResourceData, _ interface{}) diag.Diagnostics {
+								t := getTimeForTest()
+								if t.After(time.Now().Add(time.Hour * 1)) {
+									err := d.Set("special", true)
+									if err != nil {
+										panic(err)
+									}
+								}
+								return nil
+							},
+							Schema: map[string]*schema.Schema{
+								"special": {
+									Computed: true,
+									Type:     schema.TypeBool,
+									ForceNew: true,
+								},
+
+								"id": {
+									Computed: true,
+									Type:     schema.TypeString,
+								},
+							},
+						},
+					},
+				}, nil
+			},
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "random_password" "test" { }`,
+				Check:  TestCheckResourceAttr("random_password.test", "special", "false"),
+			},
+			{
+				PreConfig:          setTimeForTest(time.Now().Add(time.Hour * 2)),
+				RefreshState:       true,
+				ExpectNonEmptyPlan: true,
+				Check:              TestCheckResourceAttr("random_password.test", "special", "true"),
+			},
+			{
+				PreConfig: setTimeForTest(time.Now()),
+				Config:    `resource "random_password" "test" { }`,
+				Check:     TestCheckResourceAttr("random_password.test", "special", "false"),
+			},
+		},
+	})
+}
+
+func setTimeForTest(t time.Time) func() {
+	return func() {
+		getTimeForTest = func() time.Time {
+			return t
+		}
+	}
+}
+
+var getTimeForTest = func() time.Time {
+	return time.Now()
+}
+
 func composeImportStateCheck(fs ...ImportStateCheckFunc) ImportStateCheckFunc {
 	return func(s []*terraform.InstanceState) error {
 		for i, f := range fs {

helper/resource/teststep_validate.go

@@ -43,7 +43,10 @@ func (s TestStep) hasProviders(_ context.Context) bool {
 
 // validate ensures the TestStep is valid based on the following criteria:
 //
-//   - Config or ImportState is set.
+//   - Config or ImportState or RefreshState is set.
+//   - Config and RefreshState are not both set.
+//   - RefreshState and Destroy are not both set.
+//   - RefreshState is not the first TestStep.
 //   - Providers are not specified (ExternalProviders,
 //     ProtoV5ProviderFactories, ProtoV6ProviderFactories, ProviderFactories)
 //     if specified at the TestCase level.
@@ -58,8 +61,32 @@ func (s TestStep) validate(ctx context.Context, req testStepValidateRequest) err
 
 	logging.HelperResourceTrace(ctx, "Validating TestStep")
 
-	if s.Config == "" && !s.ImportState {
-		err := fmt.Errorf("TestStep missing Config or ImportState")
+	if s.Config == "" && !s.ImportState && !s.RefreshState {
+		err := fmt.Errorf("TestStep missing Config or ImportState or RefreshState")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
+	if s.Config != "" && s.RefreshState {
+		err := fmt.Errorf("TestStep cannot have Config and RefreshState")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
+	if s.RefreshState && s.Destroy {
+		err := fmt.Errorf("TestStep cannot have RefreshState and Destroy")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
+	if s.RefreshState && req.StepNumber == 1 {
+		err := fmt.Errorf("TestStep cannot have RefreshState as first step")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
+	if s.ImportState && s.RefreshState {
+		err := fmt.Errorf("TestStep cannot have ImportState and RefreshState in same step")
 		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
 		return err
 	}