diff --git a/README.md b/README.md
index 134a75c..50477e5 100644
--- a/README.md
+++ b/README.md
@@ -315,20 +315,22 @@ func FunctionReconciler(c reconcilers.Config) *reconcilers.ParentReconciler {
 `WithConfig` can be used to change the REST Config backing the clients. This could be to make requests to the same cluster with a user defined service account, or target an entirely different Kubernetes cluster.
 
 ```go
-func SwapRESTConfig(c reconciler.Config, rc *rest.Config) (*reconcilers.SubReconciler, error) {
-	cl, err := clusters.New(rc)
-	if err != nil {
-		return nil, err
-	}
-
+func SwapRESTConfig(rc *rest.Config) *reconcilers.SubReconciler {
 	return &reconcilers.WithConfig{
 		Reconciler: reconcilers.Sequence{
 			LookupReferenceDataReconciler(),
 			DoSomethingChildReconciler(),
 		},
 
-		Config: c.WithCluster(cl),
-	}, nil
+		Config: func(ctx context.Context, c reconciler.Config) (reconciler.Config, error ) {
+			// the rest config could also be stashed from a lookup in a SyncReconciler based on a dynamic value
+			cl, err := clusters.New(rc)
+			if err != nil {
+				return reconciler.Config{}, err
+			}
+			return c.WithCluster(cl), nil
+		}
+	}
 }
 ```
 
diff --git a/reconcilers/reconcilers.go b/reconcilers/reconcilers.go
index cf7d518..bd942c2 100644
--- a/reconcilers/reconcilers.go
+++ b/reconcilers/reconcilers.go
@@ -57,7 +57,7 @@ func (c Config) IsEmpty() bool {
 	return c == Config{}
 }
 
-// WithConfig extends the config to access a new cluster.
+// WithCluster extends the config to access a new cluster.
 func (c Config) WithCluster(cluster cluster.Cluster) Config {
 	return Config{
 		Client:    cluster.GetClient(),
@@ -1112,8 +1112,10 @@ func (r *CastParent) cast(ctx context.Context, parent client.Object) (context.Co
 // The specified config can be accessed with `RetrieveConfig(ctx)`, the original config used to
 // load the parent resource can be accessed with `RetrieveParentConfig(ctx)`.
 type WithConfig struct {
-	// Config to use for this portion of the reconciler hierarchy
-	Config Config
+	// Config to use for this portion of the reconciler hierarchy. This method is called during
+	// setup and during reconciliation, if context is needed, it should be available durring both
+	// phases.
+	Config func(context.Context, Config) (Config, error)
 
 	// Reconciler is called for each reconciler request with the parent
 	// resource being reconciled. Typically a Sequence is used to compose
@@ -1125,13 +1127,17 @@ func (r *WithConfig) SetupWithManager(ctx context.Context, mgr ctrl.Manager, bld
 	if err := r.validate(ctx); err != nil {
 		return err
 	}
-	ctx = StashConfig(ctx, r.Config)
+	c, err := r.Config(ctx, RetrieveConfig(ctx))
+	if err != nil {
+		return err
+	}
+	ctx = StashConfig(ctx, c)
 	return r.Reconciler.SetupWithManager(ctx, mgr, bldr)
 }
 
 func (r *WithConfig) validate(ctx context.Context) error {
 	// validate Config value
-	if r.Config.IsEmpty() {
+	if r.Config == nil {
 		return fmt.Errorf("Config must be defined")
 	}
 
@@ -1144,7 +1150,11 @@ func (r *WithConfig) validate(ctx context.Context) error {
 }
 
 func (r *WithConfig) Reconcile(ctx context.Context, parent client.Object) (ctrl.Result, error) {
-	ctx = StashConfig(ctx, r.Config)
+	c, err := r.Config(ctx, RetrieveConfig(ctx))
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	ctx = StashConfig(ctx, c)
 	return r.Reconciler.Reconcile(ctx, parent)
 }
 
diff --git a/reconcilers/reconcilers_test.go b/reconcilers/reconcilers_test.go
index c1b33a0..0ca68cc 100644
--- a/reconcilers/reconcilers_test.go
+++ b/reconcilers/reconcilers_test.go
@@ -1429,7 +1429,9 @@ func TestWithConfig(t *testing.T) {
 				}
 
 				return &reconcilers.WithConfig{
-					Config: c,
+					Config: func(ctx context.Context, _ reconcilers.Config) (reconcilers.Config, error) {
+						return c, nil
+					},
 					Reconciler: &reconcilers.SyncReconciler{
 						Sync: func(ctx context.Context, parent *resources.TestResource) error {
 							ac := reconcilers.RetrieveConfig(ctx)
diff --git a/reconcilers/reconcilers_validate_test.go b/reconcilers/reconcilers_validate_test.go
index 5ae3678..7dcd5fc 100644
--- a/reconcilers/reconcilers_validate_test.go
+++ b/reconcilers/reconcilers_validate_test.go
@@ -772,23 +772,17 @@ func TestWithConfig_validate(t *testing.T) {
 			name:   "valid",
 			parent: &corev1.ConfigMap{},
 			reconciler: &WithConfig{
-				Config: config,
-				Reconciler: &SyncReconciler{
-					Sync: func(ctx context.Context, parent *corev1.Secret) error {
-						return nil
-					},
+				Reconciler: &Sequence{},
+				Config: func(ctx context.Context, c Config) (Config, error) {
+					return config, nil
 				},
 			},
 		},
 		{
-			name:   "missing type",
+			name:   "missing config",
 			parent: &corev1.ConfigMap{},
 			reconciler: &WithConfig{
-				Reconciler: &SyncReconciler{
-					Sync: func(ctx context.Context, parent *corev1.Secret) error {
-						return nil
-					},
-				},
+				Reconciler: &Sequence{},
 			},
 			shouldErr: "Config must be defined",
 		},
@@ -796,8 +790,9 @@ func TestWithConfig_validate(t *testing.T) {
 			name:   "missing reconciler",
 			parent: &corev1.ConfigMap{},
 			reconciler: &WithConfig{
-				Config:     config,
-				Reconciler: nil,
+				Config: func(ctx context.Context, c Config) (Config, error) {
+					return config, nil
+				},
 			},
 			shouldErr: "Reconciler must be defined",
 		},