diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index a82f44502598..592e4efae17f 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -1318,6 +1318,20 @@ } ] }, + "io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy": { + "description": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.", + "properties": { + "whenDeleted": { + "description": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.", + "type": "string" + }, + "whenScaled": { + "description": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.", + "type": "string" + } + }, + "type": "object" + }, "io.k8s.api.apps.v1.StatefulSetSpec": { "description": "A StatefulSetSpec is the specification of a StatefulSet.", "properties": { @@ -1326,6 +1340,10 @@ "format": "int32", "type": "integer" }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy", + "description": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional" + }, "podManagementPolicy": { "description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.", "enum": [ diff --git a/api/openapi-spec/v3/apis__apps__v1_openapi.json b/api/openapi-spec/v3/apis__apps__v1_openapi.json index eb366df25668..103b6b2f8376 100644 --- a/api/openapi-spec/v3/apis__apps__v1_openapi.json +++ b/api/openapi-spec/v3/apis__apps__v1_openapi.json @@ -8822,6 +8822,20 @@ } ] }, + "io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy": { + "description": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.", + "type": "object", + "properties": { + "whenDeleted": { + "description": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.", + "type": "string" + }, + "whenScaled": { + "description": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.", + "type": "string" + } + } + }, "io.k8s.api.apps.v1.StatefulSetSpec": { "description": "A StatefulSetSpec is the specification of a StatefulSet.", "type": "object", @@ -8836,6 +8850,10 @@ "type": "integer", "format": "int32" }, + "persistentVolumeClaimRetentionPolicy": { + "description": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional", + "$ref": "#/components/schemas/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy" + }, "podManagementPolicy": { "description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.", "type": "string", diff --git a/pkg/apis/apps/fuzzer/fuzzer.go b/pkg/apis/apps/fuzzer/fuzzer.go index 5f7858560baa..c271223747a7 100644 --- a/pkg/apis/apps/fuzzer/fuzzer.go +++ b/pkg/apis/apps/fuzzer/fuzzer.go @@ -40,6 +40,17 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { if len(s.Spec.UpdateStrategy.Type) == 0 { s.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType } + if s.Spec.PersistentVolumeClaimRetentionPolicy == nil { + policies := []apps.PersistentVolumeClaimRetentionPolicyType{ + apps.RetainPersistentVolumeClaimRetentionPolicyType, + apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + choice := int32(c.Rand.Int31()) + s.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: policies[choice&1], + WhenScaled: policies[(choice>>1)&1], + } + } if s.Spec.RevisionHistoryLimit == nil { s.Spec.RevisionHistoryLimit = new(int32) *s.Spec.RevisionHistoryLimit = 10 diff --git a/pkg/apis/apps/types.go b/pkg/apis/apps/types.go index 506db6cf110f..70a841c285f4 100644 --- a/pkg/apis/apps/types.go +++ b/pkg/apis/apps/types.go @@ -97,6 +97,40 @@ type RollingUpdateStatefulSetStrategy struct { Partition int32 } +// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine +// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is +// deleted or scaled down. +type PersistentVolumeClaimRetentionPolicyType string + +const ( + // RetainPersistentVolumeClaimRetentionPolicyType is the default + // PersistentVolumeClaimRetentionPolicy and specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will not be deleted. + RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain" + // DeletePersistentVolumeClaimRetentionPolicyType specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will be deleted in the scenario specified in + // StatefulSetPersistentVolumeClaimPolicy. + DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +type StatefulSetPersistentVolumeClaimRetentionPolicy struct { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + WhenDeleted PersistentVolumeClaimRetentionPolicyType + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + WhenScaled PersistentVolumeClaimRetentionPolicyType +} + // A StatefulSetSpec is the specification of a StatefulSet. type StatefulSetSpec struct { // Replicas is the desired number of replicas of the given Template. @@ -164,6 +198,12 @@ type StatefulSetSpec struct { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional MinReadySeconds int32 + + // PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from + // the StatefulSet VolumeClaimTemplates. This requires the + // StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. + // +optional + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/pkg/apis/apps/v1/defaults.go b/pkg/apis/apps/v1/defaults.go index 8b3a0e9882ef..c6862ec7fc99 100644 --- a/pkg/apis/apps/v1/defaults.go +++ b/pkg/apis/apps/v1/defaults.go @@ -20,6 +20,8 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -117,6 +119,18 @@ func SetDefaults_StatefulSet(obj *appsv1.StatefulSet) { *obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0 } + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil { + obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1.RetainPersistentVolumeClaimRetentionPolicyType + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1.RetainPersistentVolumeClaimRetentionPolicyType + } + } + if obj.Spec.Replicas == nil { obj.Spec.Replicas = new(int32) *obj.Spec.Replicas = 1 diff --git a/pkg/apis/apps/v1/defaults_test.go b/pkg/apis/apps/v1/defaults_test.go index 65caa56d5a14..9dec309753b8 100644 --- a/pkg/apis/apps/v1/defaults_test.go +++ b/pkg/apis/apps/v1/defaults_test.go @@ -27,10 +27,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" _ "k8s.io/kubernetes/pkg/apis/apps/install" . "k8s.io/kubernetes/pkg/apis/apps/v1" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" ) @@ -193,10 +196,13 @@ func TestSetDefaultStatefulSet(t *testing.T) { } tests := []struct { - original *appsv1.StatefulSet - expected *appsv1.StatefulSet + name string + original *appsv1.StatefulSet + expected *appsv1.StatefulSet + enablePVCDeletionPolicy bool }{ - { // labels and default update strategy + { + name: "labels and default update strategy", original: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ Template: defaultTemplate, @@ -221,7 +227,8 @@ func TestSetDefaultStatefulSet(t *testing.T) { }, }, }, - { // Alternate update strategy + { + name: "Alternate update strategy", original: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ Template: defaultTemplate, @@ -246,7 +253,8 @@ func TestSetDefaultStatefulSet(t *testing.T) { }, }, }, - { // Parallel pod management policy. + { + name: "Parallel pod management policy", original: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ Template: defaultTemplate, @@ -272,7 +280,8 @@ func TestSetDefaultStatefulSet(t *testing.T) { }, }, }, - { // UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set + { + name: "UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set", original: &appsv1.StatefulSet{ Spec: appsv1.StatefulSetSpec{ Template: defaultTemplate, @@ -302,20 +311,151 @@ func TestSetDefaultStatefulSet(t *testing.T) { }, }, }, + { + name: "PVC delete policy enabled, no policy specified", + original: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: defaultTemplate, + }, + }, + expected: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: defaultLabels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &defaultReplicas, + Template: defaultTemplate, + PodManagementPolicy: appsv1.OrderedReadyPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &defaultPartition, + }, + }, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + RevisionHistoryLimit: utilpointer.Int32Ptr(10), + }, + }, + enablePVCDeletionPolicy: true, + }, + { + name: "PVC delete policy enabled, with scaledown policy specified", + original: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: defaultTemplate, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + }, + expected: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: defaultLabels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &defaultReplicas, + Template: defaultTemplate, + PodManagementPolicy: appsv1.OrderedReadyPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &defaultPartition, + }, + }, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + RevisionHistoryLimit: utilpointer.Int32Ptr(10), + }, + }, + enablePVCDeletionPolicy: true, + }, + { + name: "PVC delete policy disabled, with set deletion policy specified", + original: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: defaultTemplate, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + }, + expected: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: defaultLabels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &defaultReplicas, + Template: defaultTemplate, + PodManagementPolicy: appsv1.OrderedReadyPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &defaultPartition, + }, + }, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + RevisionHistoryLimit: utilpointer.Int32Ptr(10), + }, + }, + enablePVCDeletionPolicy: true, + }, + { + name: "PVC delete policy disabled, with policy specified", + original: &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: defaultTemplate, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + }, + expected: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: defaultLabels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &defaultReplicas, + Template: defaultTemplate, + PodManagementPolicy: appsv1.OrderedReadyPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &defaultPartition, + }, + }, + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + RevisionHistoryLimit: utilpointer.Int32Ptr(10), + }, + }, + enablePVCDeletionPolicy: false, + }, } - for i, test := range tests { - original := test.original - expected := test.expected - obj2 := roundTrip(t, runtime.Object(original)) - got, ok := obj2.(*appsv1.StatefulSet) - if !ok { - t.Errorf("(%d) unexpected object: %v", i, got) - t.FailNow() - } - if !apiequality.Semantic.DeepEqual(got.Spec, expected.Spec) { - t.Errorf("(%d) got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", i, got.Spec, expected.Spec) - } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, test.enablePVCDeletionPolicy)() + + obj2 := roundTrip(t, runtime.Object(test.original)) + got, ok := obj2.(*appsv1.StatefulSet) + if !ok { + t.Errorf("unexpected object: %v", got) + t.FailNow() + } + if !apiequality.Semantic.DeepEqual(got.Spec, test.expected.Spec) { + t.Errorf("got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", got.Spec, test.expected.Spec) + } + }) } } diff --git a/pkg/apis/apps/v1/zz_generated.conversion.go b/pkg/apis/apps/v1/zz_generated.conversion.go index e30a48fa36dd..7789fb681084 100644 --- a/pkg/apis/apps/v1/zz_generated.conversion.go +++ b/pkg/apis/apps/v1/zz_generated.conversion.go @@ -261,6 +261,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) }); err != nil { @@ -1154,6 +1164,28 @@ func Convert_apps_StatefulSetList_To_v1_StatefulSetList(in *apps.StatefulSetList return autoConvert_apps_StatefulSetList_To_v1_StatefulSetList(in, out, s) } +func autoConvert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + +func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = v1.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = v1.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + func autoConvert_v1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { return err @@ -1170,6 +1202,7 @@ func autoConvert_v1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1.StatefulSetSp } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } @@ -1189,6 +1222,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1_StatefulSetSpec(in *apps.StatefulSet } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } diff --git a/pkg/apis/apps/v1beta1/defaults.go b/pkg/apis/apps/v1beta1/defaults.go index 3670e67a7d6d..799973e3fca1 100644 --- a/pkg/apis/apps/v1beta1/defaults.go +++ b/pkg/apis/apps/v1beta1/defaults.go @@ -21,6 +21,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -46,6 +48,19 @@ func SetDefaults_StatefulSet(obj *appsv1beta1.StatefulSet) { obj.Labels = labels } } + + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil { + obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType + } + } + if obj.Spec.Replicas == nil { obj.Spec.Replicas = new(int32) *obj.Spec.Replicas = 1 diff --git a/pkg/apis/apps/v1beta1/zz_generated.conversion.go b/pkg/apis/apps/v1beta1/zz_generated.conversion.go index 4bc769c55bac..e1ba487a11e0 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/apps/v1beta1/zz_generated.conversion.go @@ -212,6 +212,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1beta1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) }); err != nil { @@ -818,6 +828,28 @@ func Convert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in *apps.StatefulSe return autoConvert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in, out, s) } +func autoConvert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + +func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = v1beta1.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = v1beta1.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + func autoConvert_v1beta1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { return err @@ -834,6 +866,7 @@ func autoConvert_v1beta1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta1.Sta } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } @@ -853,6 +886,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1beta1_StatefulSetSpec(in *apps.Statef } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } diff --git a/pkg/apis/apps/v1beta2/defaults.go b/pkg/apis/apps/v1beta2/defaults.go index f42b021b04e8..78a415d64784 100644 --- a/pkg/apis/apps/v1beta2/defaults.go +++ b/pkg/apis/apps/v1beta2/defaults.go @@ -20,6 +20,8 @@ import ( appsv1beta2 "k8s.io/api/apps/v1beta2" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -72,6 +74,18 @@ func SetDefaults_StatefulSet(obj *appsv1beta2.StatefulSet) { *obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0 } + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil { + obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1beta2.RetainPersistentVolumeClaimRetentionPolicyType + } + if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 { + obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1beta2.RetainPersistentVolumeClaimRetentionPolicyType + } + } + if obj.Spec.Replicas == nil { obj.Spec.Replicas = new(int32) *obj.Spec.Replicas = 1 diff --git a/pkg/apis/apps/v1beta2/zz_generated.conversion.go b/pkg/apis/apps/v1beta2/zz_generated.conversion.go index 7f80adb30a60..b5deccf43630 100644 --- a/pkg/apis/apps/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/apps/v1beta2/zz_generated.conversion.go @@ -282,6 +282,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1beta2.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta2.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) }); err != nil { @@ -1250,6 +1260,28 @@ func Convert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in *apps.StatefulSe return autoConvert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in, out, s) } +func autoConvert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + +func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + out.WhenDeleted = v1beta2.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted) + out.WhenScaled = v1beta2.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled) + return nil +} + +// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function. +func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error { + return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s) +} + func autoConvert_v1beta2_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta2.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { return err @@ -1266,6 +1298,7 @@ func autoConvert_v1beta2_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta2.Sta } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } @@ -1285,6 +1318,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1beta2_StatefulSetSpec(in *apps.Statef } out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.MinReadySeconds = in.MinReadySeconds + out.PersistentVolumeClaimRetentionPolicy = (*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy)) return nil } diff --git a/pkg/apis/apps/validation/validation.go b/pkg/apis/apps/validation/validation.go index 90f6265dde04..e2eedf45c5ad 100644 --- a/pkg/apis/apps/validation/validation.go +++ b/pkg/apis/apps/validation/validation.go @@ -69,6 +69,26 @@ func ValidatePodTemplateSpecForStatefulSet(template *api.PodTemplateSpec, select return allErrs } +func ValidatePersistentVolumeClaimRetentionPolicyType(policy apps.PersistentVolumeClaimRetentionPolicyType, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + switch policy { + case apps.RetainPersistentVolumeClaimRetentionPolicyType: + case apps.DeletePersistentVolumeClaimRetentionPolicyType: + default: + allErrs = append(allErrs, field.NotSupported(fldPath, policy, []string{string(apps.RetainPersistentVolumeClaimRetentionPolicyType), string(apps.DeletePersistentVolumeClaimRetentionPolicyType)})) + } + return allErrs +} + +func ValidatePersistentVolumeClaimRetentionPolicy(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if policy != nil { + allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicyType(policy.WhenDeleted, fldPath.Child("whenDeleted"))...) + allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicyType(policy.WhenScaled, fldPath.Child("whenScaled"))...) + } + return allErrs +} + // ValidateStatefulSetSpec tests if required fields in the StatefulSet spec are set. func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path, opts apivalidation.PodValidationOptions) field.ErrorList { allErrs := field.ErrorList{} @@ -108,6 +128,8 @@ func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path, op apps.OnDeleteStatefulSetStrategyType))) } + allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicy(spec.PersistentVolumeClaimRetentionPolicy, fldPath.Child("persistentVolumeClaimRetentionPolicy"))...) + allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.Replicas), fldPath.Child("replicas"))...) if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.MinReadySeconds), fldPath.Child("minReadySeconds"))...) @@ -158,11 +180,12 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) fi if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { newStatefulSetClone.Spec.MinReadySeconds = oldStatefulSet.Spec.MinReadySeconds // +k8s:verify-mutation:reason=clone } + newStatefulSetClone.Spec.PersistentVolumeClaimRetentionPolicy = oldStatefulSet.Spec.PersistentVolumeClaimRetentionPolicy // +k8s:verify-mutation:reason=clone if !apiequality.Semantic.DeepEqual(newStatefulSetClone.Spec, oldStatefulSet.Spec) { if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { - allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'minReadySeconds' and 'updateStrategy' are forbidden")) + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden")) } else { - allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy' and 'minReadySeconds' are forbidden")) + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy' and 'persistentVolumeClaimRetentionPolicy' are forbidden")) } } @@ -170,6 +193,7 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) fi if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.MinReadySeconds), field.NewPath("spec", "minReadySeconds"))...) } + allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicy(statefulSet.Spec.PersistentVolumeClaimRetentionPolicy, field.NewPath("spec", "persistentVolumeClaimRetentionPolicy"))...) return allErrs } diff --git a/pkg/apis/apps/validation/validation_test.go b/pkg/apis/apps/validation/validation_test.go index 184a1d3c3c6a..7d5d69cc9706 100644 --- a/pkg/apis/apps/validation/validation_test.go +++ b/pkg/apis/apps/validation/validation_test.go @@ -76,8 +76,10 @@ func TestValidateStatefulSet(t *testing.T) { }, } - successCases := []apps.StatefulSet{ - { + const enableStatefulSetAutoDeletePVC = "[enable StatefulSetAutoDeletePVC]" + + successCases := map[string]apps.StatefulSet{ + "alpha name": { ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, @@ -86,7 +88,7 @@ func TestValidateStatefulSet(t *testing.T) { UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, - { + "alphanumeric name": { ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, @@ -95,7 +97,7 @@ func TestValidateStatefulSet(t *testing.T) { UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, - { + "parallel pod management": { ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.ParallelPodManagement, @@ -104,7 +106,7 @@ func TestValidateStatefulSet(t *testing.T) { UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, - { + "ordered ready pod management": { ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, @@ -113,7 +115,7 @@ func TestValidateStatefulSet(t *testing.T) { UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType}, }, }, - { + "update strategy": { ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, Spec: apps.StatefulSetSpec{ PodManagementPolicy: apps.OrderedReadyPodManagement, @@ -127,11 +129,29 @@ func TestValidateStatefulSet(t *testing.T) { }()}, }, }, + "PVC policy " + enableStatefulSetAutoDeletePVC: { + ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + }, + }, } - for i, successCase := range successCases { - t.Run("success case "+strconv.Itoa(i), func(t *testing.T) { - if errs := ValidateStatefulSet(&successCase, corevalidation.PodValidationOptions{}); len(errs) != 0 { + for name, successCase := range successCases { + name := name + set := successCase + t.Run("success case "+name, func(t *testing.T) { + if strings.Contains(name, enableStatefulSetAutoDeletePVC) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + } + if errs := ValidateStatefulSet(&set, corevalidation.PodValidationOptions{}); len(errs) != 0 { t.Errorf("expected success: %v", errs) } }) @@ -352,10 +372,35 @@ func TestValidateStatefulSet(t *testing.T) { UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, }, }, + "empty PersistentVolumeClaimRetentionPolicy " + enableStatefulSetAutoDeletePVC: { + ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{}, + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + "invalid PersistentVolumeClaimRetentionPolicy " + enableStatefulSetAutoDeletePVC: { + ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.PersistentVolumeClaimRetentionPolicyType("invalid-delete-policy"), + }, + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, } for k, v := range errorCases { t.Run(k, func(t *testing.T) { + if strings.Contains(k, enableStatefulSetAutoDeletePVC) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + } errs := ValidateStatefulSet(&v, corevalidation.PodValidationOptions{}) if len(errs) == 0 { t.Errorf("expected failure for %s", k) @@ -378,6 +423,9 @@ func TestValidateStatefulSet(t *testing.T) { field != "spec.updateStrategy.rollingUpdate" && field != "spec.updateStrategy.rollingUpdate.partition" && field != "spec.podManagementPolicy" && + field != "spec.persistentVolumeClaimRetentionPolicy" && + field != "spec.persistentVolumeClaimRetentionPolicy.whenDeleted" && + field != "spec.persistentVolumeClaimRetentionPolicy.whenScaled" && field != "spec.template.spec.activeDeadlineSeconds" { t.Errorf("%s: missing prefix for: %v", k, errs[i]) } @@ -748,6 +796,53 @@ func TestValidateStatefulSetUpdate(t *testing.T) { }, }, }, + { + old: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: addContainersValidTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + update: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + }, + }, + }, + { + old: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: addContainersValidTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + }, + }, + update: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + }, } for i, successCase := range successCases { diff --git a/pkg/apis/apps/zz_generated.deepcopy.go b/pkg/apis/apps/zz_generated.deepcopy.go index 482d542445a0..b496f6bff528 100644 --- a/pkg/apis/apps/zz_generated.deepcopy.go +++ b/pkg/apis/apps/zz_generated.deepcopy.go @@ -712,6 +712,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy { + if in == nil { + return nil + } + out := new(StatefulSetPersistentVolumeClaimRetentionPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in @@ -734,6 +750,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = new(int32) **out = **in } + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = new(StatefulSetPersistentVolumeClaimRetentionPolicy) + **out = **in + } return } diff --git a/pkg/controller/statefulset/stateful_pod_control.go b/pkg/controller/statefulset/stateful_pod_control.go index 4bfc5c847b13..0d045763e33f 100644 --- a/pkg/controller/statefulset/stateful_pod_control.go +++ b/pkg/controller/statefulset/stateful_pod_control.go @@ -27,68 +27,115 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" errorutils "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" - appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" ) -// StatefulPodControlInterface defines the interface that StatefulSetController uses to create, update, and delete Pods, +// StatefulPodControlObjectManager abstracts the manipulation of Pods and PVCs. The real controller implements this +// with a clientset for writes and listers for reads; for tests we provide stubs. +type StatefulPodControlObjectManager interface { + CreatePod(ctx context.Context, pod *v1.Pod) error + GetPod(namespace, podName string) (*v1.Pod, error) + UpdatePod(pod *v1.Pod) error + DeletePod(pod *v1.Pod) error + CreateClaim(claim *v1.PersistentVolumeClaim) error + GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) + UpdateClaim(claim *v1.PersistentVolumeClaim) error +} + +// StatefulPodControl defines the interface that StatefulSetController uses to create, update, and delete Pods, // and to update the Status of a StatefulSet. It follows the design paradigms used for PodControl, but its // implementation provides for PVC creation, ordered Pod creation, ordered Pod termination, and Pod identity enforcement. -// Like controller.PodControlInterface, it is implemented as an interface to provide for testing fakes. -type StatefulPodControlInterface interface { - // CreateStatefulPod create a Pod in a StatefulSet. Any PVCs necessary for the Pod are created prior to creating - // the Pod. If the returned error is nil the Pod and its PVCs have been created. - CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error - // UpdateStatefulPod Updates a Pod in a StatefulSet. If the Pod already has the correct identity and stable - // storage this method is a no-op. If the Pod must be mutated to conform to the Set, it is mutated and updated. - // pod is an in-out parameter, and any updates made to the pod are reflected as mutations to this parameter. If - // the create is successful, the returned error is nil. - UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error - // DeleteStatefulPod deletes a Pod in a StatefulSet. The pods PVCs are not deleted. If the delete is successful, - // the returned error is nil. - DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error -} - -func NewRealStatefulPodControl( +// Manipulation of objects is provided through objectMgr, which allows the k8s API to be mocked out for testing. +type StatefulPodControl struct { + objectMgr StatefulPodControlObjectManager + recorder record.EventRecorder +} + +// NewStatefulPodControl constructs a StatefulPodControl using a realStatefulPodControlObjectManager with the given +// clientset, listers and EventRecorder. +func NewStatefulPodControl( client clientset.Interface, - setLister appslisters.StatefulSetLister, podLister corelisters.PodLister, - pvcLister corelisters.PersistentVolumeClaimLister, + claimLister corelisters.PersistentVolumeClaimLister, recorder record.EventRecorder, -) StatefulPodControlInterface { - return &realStatefulPodControl{client, setLister, podLister, pvcLister, recorder} +) *StatefulPodControl { + return &StatefulPodControl{&realStatefulPodControlObjectManager{client, podLister, claimLister}, recorder} } -// realStatefulPodControl implements StatefulPodControlInterface using a clientset.Interface to communicate with the -// API server. The struct is package private as the internal details are irrelevant to importing packages. -type realStatefulPodControl struct { - client clientset.Interface - setLister appslisters.StatefulSetLister - podLister corelisters.PodLister - pvcLister corelisters.PersistentVolumeClaimLister - recorder record.EventRecorder +// NewStatefulPodControlFromManager creates a StatefulPodControl using the given StatefulPodControlObjectManager and recorder. +func NewStatefulPodControlFromManager(om StatefulPodControlObjectManager, recorder record.EventRecorder) *StatefulPodControl { + return &StatefulPodControl{om, recorder} +} + +// realStatefulPodControlObjectManager uses a clientset.Interface and listers. +type realStatefulPodControlObjectManager struct { + client clientset.Interface + podLister corelisters.PodLister + claimLister corelisters.PersistentVolumeClaimLister +} + +func (om *realStatefulPodControlObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error { + _, err := om.client.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + return err +} + +func (om *realStatefulPodControlObjectManager) GetPod(namespace, podName string) (*v1.Pod, error) { + return om.podLister.Pods(namespace).Get(podName) +} + +func (om *realStatefulPodControlObjectManager) UpdatePod(pod *v1.Pod) error { + _, err := om.client.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) + return err } -func (spc *realStatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error { +func (om *realStatefulPodControlObjectManager) DeletePod(pod *v1.Pod) error { + return om.client.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) +} + +func (om *realStatefulPodControlObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim) error { + _, err := om.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), claim, metav1.CreateOptions{}) + return err +} + +func (om *realStatefulPodControlObjectManager) GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) { + return om.claimLister.PersistentVolumeClaims(namespace).Get(claimName) +} + +func (om *realStatefulPodControlObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim) error { + _, err := om.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(context.TODO(), claim, metav1.UpdateOptions{}) + return err +} + +func (spc *StatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error { // Create the Pod's PVCs prior to creating the Pod if err := spc.createPersistentVolumeClaims(set, pod); err != nil { spc.recordPodEvent("create", set, pod, err) return err } // If we created the PVCs attempt to create the Pod - _, err := spc.client.CoreV1().Pods(set.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + err := spc.objectMgr.CreatePod(ctx, pod) // sink already exists errors if apierrors.IsAlreadyExists(err) { return err } + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + // Set PVC policy as much as is possible at this point. + if err := spc.UpdatePodClaimForRetentionPolicy(set, pod); err != nil { + spc.recordPodEvent("update", set, pod, err) + return err + } + } spc.recordPodEvent("create", set, pod, err) return err } -func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { +func (spc *StatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { attemptedUpdate := false err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { // assume the Pod is consistent @@ -108,6 +155,21 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod return err } } + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + // if the Pod's PVCs are not consistent with the StatefulSet's PVC deletion policy, update the PVC + // and dirty the pod. + if match, err := spc.ClaimsMatchRetentionPolicy(set, pod); err != nil { + spc.recordPodEvent("update", set, pod, err) + return err + } else if !match { + if err := spc.UpdatePodClaimForRetentionPolicy(set, pod); err != nil { + spc.recordPodEvent("update", set, pod, err) + return err + } + consistent = false + } + } + // if the Pod is not dirty, do nothing if consistent { return nil @@ -115,16 +177,17 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod attemptedUpdate = true // commit the update, retrying on conflicts - _, updateErr := spc.client.CoreV1().Pods(set.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}) + + updateErr := spc.objectMgr.UpdatePod(pod) if updateErr == nil { return nil } - if updated, err := spc.podLister.Pods(set.Namespace).Get(pod.Name); err == nil { + if updated, err := spc.objectMgr.GetPod(set.Namespace, pod.Name); err == nil { // make a copy so we don't mutate the shared cache pod = updated.DeepCopy() } else { - utilruntime.HandleError(fmt.Errorf("error getting updated Pod %s/%s from lister: %v", set.Namespace, pod.Name, err)) + utilruntime.HandleError(fmt.Errorf("error getting updated Pod %s/%s: %w", set.Namespace, pod.Name, err)) } return updateErr @@ -135,15 +198,92 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod return err } -func (spc *realStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { - err := spc.client.CoreV1().Pods(set.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) +func (spc *StatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { + err := spc.objectMgr.DeletePod(pod) spc.recordPodEvent("delete", set, pod, err) return err } +// ClaimsMatchRetentionPolicy returns false if the PVCs for pod are not consistent with set's PVC deletion policy. +// An error is returned if something is not consistent. This is expected if the pod is being otherwise updated, +// but a problem otherwise (see usage of this method in UpdateStatefulPod). +func (spc *StatefulPodControl) ClaimsMatchRetentionPolicy(set *apps.StatefulSet, pod *v1.Pod) (bool, error) { + ordinal := getOrdinal(pod) + templates := set.Spec.VolumeClaimTemplates + for i := range templates { + claimName := getPersistentVolumeClaimName(set, &templates[i], ordinal) + claim, err := spc.objectMgr.GetClaim(set.Namespace, claimName) + switch { + case apierrors.IsNotFound(err): + klog.V(4).Infof("Expected claim %s missing, continuing to pick up in next iteration", claimName) + case err != nil: + return false, fmt.Errorf("Could not retrieve claim %s for %s when checking PVC deletion policy", claimName, pod.Name) + default: + if !claimOwnerMatchesSetAndPod(claim, set, pod) { + return false, nil + } + } + } + return true, nil +} + +// UpdatePodClaimForRetentionPolicy updates the PVCs used by pod to match the PVC deletion policy of set. +func (spc *StatefulPodControl) UpdatePodClaimForRetentionPolicy(set *apps.StatefulSet, pod *v1.Pod) error { + ordinal := getOrdinal(pod) + templates := set.Spec.VolumeClaimTemplates + for i := range templates { + claimName := getPersistentVolumeClaimName(set, &templates[i], ordinal) + claim, err := spc.objectMgr.GetClaim(set.Namespace, claimName) + switch { + case apierrors.IsNotFound(err): + klog.V(4).Infof("Expected claim %s missing, continuing to pick up in next iteration.") + case err != nil: + return fmt.Errorf("Could not retrieve claim %s not found for %s when checking PVC deletion policy: %w", claimName, pod.Name, err) + default: + if !claimOwnerMatchesSetAndPod(claim, set, pod) { + needsUpdate := updateClaimOwnerRefForSetAndPod(claim, set, pod) + if needsUpdate { + err := spc.objectMgr.UpdateClaim(claim) + if err != nil { + return fmt.Errorf("Could not update claim %s for delete policy ownerRefs: %w", claimName, err) + } + } + } + } + } + return nil +} + +// PodClaimIsStale returns true for a stale PVC that should block pod creation. If the scaling +// policy is deletion, and a PVC has an ownerRef that does not match the pod, the PVC is stale. This +// includes pods whose UID has not been created. +func (spc *StatefulPodControl) PodClaimIsStale(set *apps.StatefulSet, pod *v1.Pod) (bool, error) { + policy := getPersistentVolumeClaimRetentionPolicy(set) + if policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType { + // PVCs are meant to be reused and so can't be stale. + return false, nil + } + for _, claim := range getPersistentVolumeClaims(set, pod) { + pvc, err := spc.objectMgr.GetClaim(claim.Namespace, claim.Name) + switch { + case apierrors.IsNotFound(err): + // If the claim doesn't exist yet, it can't be stale. + continue + case err != nil: + return false, err + case err == nil: + // A claim is stale if it doesn't match the pod's UID, including if the pod has no UID. + if hasStaleOwnerRef(pvc, pod) { + return true, nil + } + } + } + return false, nil +} + // recordPodEvent records an event for verb applied to a Pod in a StatefulSet. If err is nil the generated event will // have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a reason of v1.EventTypeWarning. -func (spc *realStatefulPodControl) recordPodEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, err error) { +func (spc *StatefulPodControl) recordPodEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, err error) { if err == nil { reason := fmt.Sprintf("Successful%s", strings.Title(verb)) message := fmt.Sprintf("%s Pod %s in StatefulSet %s successful", @@ -160,7 +300,7 @@ func (spc *realStatefulPodControl) recordPodEvent(verb string, set *apps.Statefu // recordClaimEvent records an event for verb applied to the PersistentVolumeClaim of a Pod in a StatefulSet. If err is // nil the generated event will have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a // reason of v1.EventTypeWarning. -func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, err error) { +func (spc *StatefulPodControl) recordClaimEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, err error) { if err == nil { reason := fmt.Sprintf("Successful%s", strings.Title(verb)) message := fmt.Sprintf("%s Claim %s Pod %s in StatefulSet %s success", @@ -178,13 +318,13 @@ func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.State // set. If all of the claims for Pod are successfully created, the returned error is nil. If creation fails, this method // may be called again until no error is returned, indicating the PersistentVolumeClaims for pod are consistent with // set's Spec. -func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { +func (spc *StatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { var errs []error for _, claim := range getPersistentVolumeClaims(set, pod) { - pvc, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + pvc, err := spc.objectMgr.GetClaim(claim.Namespace, claim.Name) switch { case apierrors.IsNotFound(err): - _, err := spc.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), &claim, metav1.CreateOptions{}) + err := spc.objectMgr.CreateClaim(&claim) if err != nil { errs = append(errs, fmt.Errorf("failed to create PVC %s: %s", claim.Name, err)) } @@ -203,5 +343,3 @@ func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.Statef } return errorutils.NewAggregate(errs) } - -var _ StatefulPodControlInterface = &realStatefulPodControl{} diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go index 82e535707ed1..4dcdfe90e3d1 100644 --- a/pkg/controller/statefulset/stateful_pod_control_test.go +++ b/pkg/controller/statefulset/stateful_pod_control_test.go @@ -19,23 +19,27 @@ package statefulset import ( "context" "errors" + "fmt" "strings" "testing" "time" + apps "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - + "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/kubernetes/fake" + corelisters "k8s.io/client-go/listers/core/v1" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - corelisters "k8s.io/client-go/listers/core/v1" + featuregatetesting "k8s.io/component-base/featuregate/testing" _ "k8s.io/kubernetes/pkg/apis/apps/install" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" ) func TestStatefulPodControlCreatesPods(t *testing.T) { @@ -43,14 +47,15 @@ func TestStatefulPodControlCreatesPods(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource) }) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) + claimIndexer.Add(create.GetObject()) return true, create.GetObject(), nil }) fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { @@ -83,7 +88,7 @@ func TestStatefulPodControlCreatePodExists(t *testing.T) { pvcIndexer.Add(&pvc) } pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -110,7 +115,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -131,7 +136,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { } } } -func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) { +func TestStatefulPodControlCreatePodPVCDeleting(t *testing.T) { recorder := record.NewFakeRecorder(10) set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) @@ -145,7 +150,7 @@ func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) { pvcIndexer.Add(&pvc) } pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -184,7 +189,7 @@ func TestStatefulPodControlCreatePodPvcGetFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := &fakeIndexer{getError: errors.New("API server down")} pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -213,7 +218,7 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -232,7 +237,6 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) { } else if !strings.Contains(events[1], v1.EventTypeWarning) { t.Errorf("Found unexpected non-warning event %s", events[1]) - } } @@ -241,7 +245,14 @@ func TestStatefulPodControlNoOpUpdate(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + claims := getPersistentVolumeClaims(set, pod) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { t.Error("no-op update should not make any client invocation") return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem")) @@ -260,7 +271,9 @@ func TestStatefulPodControlUpdatesIdentity(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := fake.NewSimpleClientset(set, pod) - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) var updated *v1.Pod fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { update := action.(core.UpdateAction) @@ -287,12 +300,14 @@ func TestStatefulPodControlUpdateIdentityFailure(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) gooPod := newStatefulSetPod(set, 0) gooPod.Name = "goo-0" - indexer.Add(gooPod) - podLister := corelisters.NewPodLister(indexer) - control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) + podIndexer.Add(gooPod) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { pod.Name = "goo-0" return true, nil, apierrors.NewInternalError(errors.New("API server down")) @@ -319,7 +334,7 @@ func TestStatefulPodControlUpdatesPodStorage(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) pvcs := getPersistentVolumeClaims(set, pod) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) for i := range pod.Spec.Volumes { @@ -366,7 +381,7 @@ func TestStatefulPodControlUpdatePodStorageFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) pvcs := getPersistentVolumeClaims(set, pod) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) for i := range pod.Spec.Volumes { @@ -401,12 +416,19 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(podIndexer) gooPod := newStatefulSetPod(set, 0) - gooPod.Name = "goo-0" - indexer.Add(gooPod) - podLister := corelisters.NewPodLister(indexer) - control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) + gooPod.Labels[apps.StatefulSetPodNameLabel] = "goo-starts" + podIndexer.Add(gooPod) + claims := getPersistentVolumeClaims(set, gooPod) + for k := range claims { + claim := claims[k] + claimIndexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) conflict := false fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { update := action.(core.UpdateAction) @@ -417,7 +439,7 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { return true, update.GetObject(), nil }) - pod.Name = "goo-0" + pod.Labels[apps.StatefulSetPodNameLabel] = "goo-0" if err := control.UpdateStatefulPod(set, pod); err != nil { t.Errorf("Successful update returned an error: %s", err) } @@ -437,7 +459,7 @@ func TestStatefulPodControlDeletesStatefulPod(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + control := NewStatefulPodControl(fakeClient, nil, nil, recorder) fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, nil, nil }) @@ -457,7 +479,7 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + control := NewStatefulPodControl(fakeClient, nil, nil, recorder) fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -472,6 +494,344 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) { } } +func TestStatefulPodControlClaimsMatchDeletionPolcy(t *testing.T) { + // The claimOwnerMatchesSetAndPod is tested exhaustively in stateful_set_utils_test; this + // test is for the wiring to the method tested there. + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + set := newStatefulSet(3) + pod := newStatefulSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (retain): %v", err) + } else if !matches { + t.Error("Unexpected non-match for ClaimsMatchRetentionPolicy (retain)") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (set deletion): %v", err) + } else if matches { + t.Error("Unexpected match for ClaimsMatchRetentionPolicy (set deletion)") + } +} + +func TestStatefulPodControlUpdatePodClaimForRetentionPolicy(t *testing.T) { + // All the update conditions are tested exhaustively in stateful_set_utils_test. This + // tests the wiring from the pod control to that method. + testFn := func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + set := newStatefulSet(3) + set.GetObjectMeta().SetUID("set-123") + pod := newStatefulSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if err := control.UpdatePodClaimForRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err) + } + expectRef := utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) + for k := range claims { + claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name) + if err != nil { + t.Errorf("Unexpected error getting Claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if hasOwnerRef(claim, set) != expectRef { + t.Errorf("Claim %s/%s bad set owner ref", claim.Namespace, claim.Name) + } + } + } + t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t) + }) + t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t) + }) +} + +func TestPodClaimIsStale(t *testing.T) { + const missing = "missing" + const exists = "exists" + const stale = "stale" + const withRef = "with-ref" + testCases := []struct { + name string + claimStates []string + expected bool + skipPodUID bool + }{ + { + name: "all missing", + claimStates: []string{missing, missing}, + expected: false, + }, + { + name: "no claims", + claimStates: []string{}, + expected: false, + }, + { + name: "exists", + claimStates: []string{missing, exists}, + expected: false, + }, + { + name: "all refs", + claimStates: []string{withRef, withRef}, + expected: false, + }, + { + name: "stale & exists", + claimStates: []string{stale, exists}, + expected: true, + }, + { + name: "stale & missing", + claimStates: []string{stale, missing}, + expected: true, + }, + { + name: "withRef & stale", + claimStates: []string{withRef, stale}, + expected: true, + }, + { + name: "withRef, no UID", + claimStates: []string{withRef}, + skipPodUID: true, + expected: true, + }, + } + for _, tc := range testCases { + set := apps.StatefulSet{} + set.Name = "set" + set.Namespace = "default" + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + set.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"key": "value"}} + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i, claimState := range tc.claimStates { + claim := v1.PersistentVolumeClaim{} + claim.Name = fmt.Sprintf("claim-%d", i) + set.Spec.VolumeClaimTemplates = append(set.Spec.VolumeClaimTemplates, claim) + claim.Name = fmt.Sprintf("%s-set-3", claim.Name) + claim.Namespace = set.Namespace + switch claimState { + case missing: + // Do nothing, the claim shouldn't exist. + case exists: + claimIndexer.Add(&claim) + case stale: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("stale")}, + }) + claimIndexer.Add(&claim) + case withRef: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("123")}, + }) + claimIndexer.Add(&claim) + } + } + pod := v1.Pod{} + pod.Name = "set-3" + if !tc.skipPodUID { + pod.SetUID("123") + } + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(&fake.Clientset{}, nil, claimLister, &noopRecorder{}) + expected := tc.expected + // Note that the error isn't / can't be tested. + if stale, _ := control.PodClaimIsStale(&set, &pod); stale != expected { + t.Errorf("unexpected stale for %s", tc.name) + } + } +} + +func TestStatefulPodControlRetainDeletionPolicyUpdate(t *testing.T) { + testFn := func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) < 1 { + t.Errorf("Unexpected missing PVCs") + } + for k := range claims { + claim := claims[k] + setOwnerRef(&claim, set, &set.TypeMeta) // This ownerRef should be removed in the update. + claimIndexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + for k := range claims { + claim := claims[k] + if hasOwnerRef(&claim, set) { + t.Errorf("ownerRef not removed: %s/%s", claim.Namespace, claim.Name) + } + } + events := collectEvents(recorder.Events) + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if eventCount := len(events); eventCount != 1 { + t.Errorf("delete failed: got %d events, but want 1", eventCount) + } + } else { + if len(events) != 0 { + t.Errorf("delete failed: expected no events, but got %v", events) + } + } + } + t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t) + }) + t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t) + }) +} + +func TestStatefulPodControlRetentionPolicyUpdate(t *testing.T) { + // Only applicable when the feature gate is on; the off case is tested in TestStatefulPodControlRetainRetentionPolicyUpdate. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + podLister := corelisters.NewPodLister(podIndexer) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 1", eventCount) + } +} + +func TestStatefulPodControlRetentionPolicyUpdateMissingClaims(t *testing.T) { + // Only applicable when the feature gate is on. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Error("Unexpected error on pod update when PVCs are missing") + } + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Expected update to succeed, saw error %v", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 2", eventCount) + } + if !strings.Contains(events[0], "SuccessfulUpdate") { + t.Errorf("expected first event to be a successful update: %s", events[1]) + } +} + func collectEvents(source <-chan string) []string { done := false events := make([]string, 0) diff --git a/pkg/controller/statefulset/stateful_set.go b/pkg/controller/statefulset/stateful_set.go index 7f5f1174ddbc..6c1d1ee7fac1 100644 --- a/pkg/controller/statefulset/stateful_set.go +++ b/pkg/controller/statefulset/stateful_set.go @@ -91,9 +91,8 @@ func NewStatefulSetController( ssc := &StatefulSetController{ kubeClient: kubeClient, control: NewDefaultStatefulSetControl( - NewRealStatefulPodControl( + NewStatefulPodControl( kubeClient, - setInformer.Lister(), podInformer.Lister(), pvcInformer.Lister(), recorder), @@ -352,10 +351,14 @@ func (ssc *StatefulSetController) getStatefulSetsForPod(pod *v1.Pod) []*apps.Sta if len(sets) > 1 { // ControllerRef will ensure we don't do anything crazy, but more than one // item in this list nevertheless constitutes user error. + setNames := []string{} + for _, s := range sets { + setNames = append(setNames, s.Name) + } utilruntime.HandleError( fmt.Errorf( - "user error: more than one StatefulSet is selecting pods with labels: %+v", - pod.Labels)) + "user error: more than one StatefulSet is selecting pods with labels: %+v. Sets: %v", + pod.Labels, setNames)) } return sets } diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 7f653d13c20e..a353e86bae08 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -54,7 +54,7 @@ type StatefulSetControlInterface interface { // to update the status of StatefulSets. You should use an instance returned from NewRealStatefulPodControl() for any // scenario other than testing. func NewDefaultStatefulSetControl( - podControl StatefulPodControlInterface, + podControl *StatefulPodControl, statusUpdater StatefulSetStatusUpdaterInterface, controllerHistory history.Interface, recorder record.EventRecorder) StatefulSetControlInterface { @@ -62,7 +62,7 @@ func NewDefaultStatefulSetControl( } type defaultStatefulSetControl struct { - podControl StatefulPodControlInterface + podControl *StatefulPodControl statusUpdater StatefulSetStatusUpdaterInterface controllerHistory history.Interface recorder record.EventRecorder @@ -333,7 +333,6 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // if the ordinal of the pod is within the range of the current number of replicas, // insert it at the indirection of its ordinal replicas[ord] = pods[i] - } else if ord >= replicaCount { // if the ordinal is greater than the number of replicas add it to the condemned list condemned = append(condemned, pods[i]) @@ -418,6 +417,14 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } // If we find a Pod that has not been created we create the Pod if !isCreated(replicas[i]) { + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if isStale, err := ssc.podControl.PodClaimIsStale(set, replicas[i]); err != nil { + return &status, err + } else if isStale { + // If a pod has a stale PVC, no more work can be done this round. + return &status, err + } + } if err := ssc.podControl.CreateStatefulPod(ctx, set, replicas[i]); err != nil { return &status, err } @@ -428,7 +435,6 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( if getPodRevision(replicas[i]) == updateRevision.Name { status.UpdatedReplicas++ } - // if the set does not allow bursting, return immediately if monotonic { return &status, nil @@ -471,7 +477,16 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( return &status, nil } // Enforce the StatefulSet invariants - if identityMatches(set, replicas[i]) && storageMatches(set, replicas[i]) { + retentionMatch := true + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + var err error + retentionMatch, err = ssc.podControl.ClaimsMatchRetentionPolicy(updateSet, replicas[i]) + // An error is expected if the pod is not yet fully updated, and so return is treated as matching. + if err != nil { + retentionMatch = true + } + } + if identityMatches(set, replicas[i]) && storageMatches(set, replicas[i]) && retentionMatch { continue } // Make a deep copy so we don't mutate the shared cache @@ -481,6 +496,19 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } } + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + // Ensure ownerRefs are set correctly for the condemned pods. + for i := range condemned { + if matchPolicy, err := ssc.podControl.ClaimsMatchRetentionPolicy(updateSet, condemned[i]); err != nil { + return &status, err + } else if !matchPolicy { + if err := ssc.podControl.UpdatePodClaimForRetentionPolicy(updateSet, condemned[i]); err != nil { + return &status, err + } + } + } + } + // At this point, all of the current Replicas are Running, Ready and Available, we can consider termination. // We will wait for all predecessors to be Running and Ready prior to attempting a deletion. // We will terminate Pods in a monotonically decreasing order over [len(pods),set.Spec.Replicas). diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index 098cf321f795..f9d65e6da8ad 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -34,17 +34,16 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" appsinformers "k8s.io/client-go/informers/apps/v1" - coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" featuregatetesting "k8s.io/component-base/featuregate/testing" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller" @@ -52,13 +51,14 @@ import ( "k8s.io/kubernetes/pkg/features" ) -type invariantFunc func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error +type invariantFunc func(set *apps.StatefulSet, om *fakeObjectManager) error -func setupController(client clientset.Interface) (*fakeStatefulPodControl, *fakeStatefulSetStatusUpdater, StatefulSetControlInterface, chan struct{}) { +func setupController(client clientset.Interface) (*fakeObjectManager, *fakeStatefulSetStatusUpdater, StatefulSetControlInterface, chan struct{}) { informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + om := newFakeObjectManager(informerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) - recorder := record.NewFakeRecorder(10) + recorder := &noopRecorder{} ssc := NewDefaultStatefulSetControl(spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder) stop := make(chan struct{}) @@ -69,7 +69,7 @@ func setupController(client clientset.Interface) (*fakeStatefulPodControl, *fake informerFactory.Core().V1().Pods().Informer().HasSynced, informerFactory.Apps().V1().ControllerRevisions().Informer().HasSynced, ) - return spc, ssu, ssc, stop + return om, ssu, ssc, stop } func burst(set *apps.StatefulSet) *apps.StatefulSet { @@ -82,6 +82,64 @@ func setMinReadySeconds(set *apps.StatefulSet, minReadySeconds int32) *apps.Stat return set } +func runTestOverPVCRetentionPolicies(t *testing.T, testName string, testFn func(*testing.T, *apps.StatefulSetPersistentVolumeClaimRetentionPolicy)) { + subtestName := "StatefulSetAutoDeletePVCDisabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t, &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }) + }) + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } { + subtestName := pvcDeletePolicyString(policy) + "/StatefulSetAutoDeletePVCEnabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t, policy) + }) + } +} + +func pvcDeletePolicyString(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) string { + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + return "Retain" + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + return "SetDeleteOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + return "ScaleDownOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + return "Delete" + } + return "invalid" +} + func TestStatefulSetControl(t *testing.T) { simpleSetFn := func() *apps.StatefulSet { return newStatefulSet(3) } largeSetFn := func() *apps.StatefulSet { return newStatefulSet(5) } @@ -106,17 +164,24 @@ func TestStatefulSetControl(t *testing.T) { if i := strings.LastIndex(fnName, "."); i != -1 { fnName = fnName[i+1:] } - t.Run( + testObj := testCase.obj + testFn := testCase.fn + runTestOverPVCRetentionPolicies( + t, fmt.Sprintf("%s/Monotonic", fnName), - func(t *testing.T) { - testCase.fn(t, testCase.obj(), assertMonotonicInvariants) + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := testObj() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertMonotonicInvariants) }, ) - t.Run( + runTestOverPVCRetentionPolicies( + t, fmt.Sprintf("%s/Burst", fnName), - func(t *testing.T) { - set := burst(testCase.obj()) - testCase.fn(t, set, assertBurstInvariants) + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := burst(testObj()) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertBurstInvariants) }, ) } @@ -124,14 +189,14 @@ func TestStatefulSetControl(t *testing.T) { func CreatesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -148,18 +213,18 @@ func CreatesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) func ScalesUp(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } *set.Spec.Replicas = 4 - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to scale StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -176,16 +241,27 @@ func ScalesUp(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { func ScalesDown(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to scale StatefulSet : %s", err) } + + // Check updated set. + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } if set.Status.Replicas != 0 { t.Error("Failed to scale statefulset to 0 replicas") } @@ -199,14 +275,14 @@ func ScalesDown(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -217,48 +293,62 @@ func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) if err != nil { t.Error(err) } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + for _, pod := range pods { + podClaims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + if _, found := podClaims[claim.Name]; found { + if hasOwnerRef(claim, pod) { + t.Errorf("Unexpected ownerRef on %s", claim.Name) + } + } + } + } sort.Sort(ascendingOrdinal(pods)) - spc.podsIndexer.Delete(pods[0]) - spc.podsIndexer.Delete(pods[2]) - spc.podsIndexer.Delete(pods[4]) + om.podsIndexer.Delete(pods[0]) + om.podsIndexer.Delete(pods[2]) + om.podsIndexer.Delete(pods[4]) for i := 0; i < 5; i += 2 { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } - if pods, err = spc.setPodRunning(set, i); err != nil { + if pods, err = om.setPodRunning(set, i); err != nil { t.Error(err) } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } - if _, err = spc.setPodReady(set, i); err != nil { + if _, err = om.setPodReady(set, i); err != nil { t.Error(err) } } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -269,35 +359,35 @@ func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) func RecreatesFailedPod(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset() - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } pods[0].Status.Phase = v1.PodFailed - spc.podsIndexer.Update(pods[0]) + om.podsIndexer.Update(pods[0]) if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -308,18 +398,23 @@ func RecreatesFailedPod(t *testing.T, set *apps.StatefulSet, invariants invarian func CreatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - spc.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + om.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSetControl did not return InternalError found %s", err) } - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + // Update so set.Status is set for the next scaleUpStatefulSetControl call. + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -336,16 +431,16 @@ func CreatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - spc.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + om.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) // have to have 1 successful loop first - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Fatalf("Unexpected error: %v", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -360,7 +455,7 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF } // now mutate a pod's identity - pods, err := spc.podsLister.List(labels.Everything()) + pods, err := om.podsLister.List(labels.Everything()) if err != nil { t.Fatalf("Error listing pods: %v", err) } @@ -369,7 +464,7 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF } sort.Sort(ascendingOrdinal(pods)) pods[0].Name = "goo-0" - spc.podsIndexer.Update(pods[0]) + om.podsIndexer.Update(pods[0]) // now it should fail if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { @@ -379,18 +474,23 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF func UpdateSetStatusFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, ssu, ssc, stop := setupController(client) + om, ssu, ssc, stop := setupController(client) defer close(stop) ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSetControl did not return InternalError found %s", err) } - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + // Update so set.Status is set for the next scaleUpStatefulSetControl call. + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -407,43 +507,43 @@ func UpdateSetStatusFailure(t *testing.T, set *apps.StatefulSet, invariants inva func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } pods[0].Status.Phase = v1.PodFailed - spc.podsIndexer.Update(pods[0]) - spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + om.podsIndexer.Update(pods[0]) + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSet failed to %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -453,41 +553,49 @@ func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants in } func TestStatefulSetControlScaleDownDeleteError(t *testing.T) { - invariants := assertMonotonicInvariants - set := newStatefulSet(3) - client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) - defer close(stop) - - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn up StatefulSet : %s", err) - } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } - *set.Spec.Replicas = 0 - spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { - t.Errorf("StatefulSetControl failed to throw error on delete %s", err) - } - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn down StatefulSet %s", err) - } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } - if set.Status.Replicas != 0 { - t.Error("Failed to scale statefulset to 0 replicas") - } - if set.Status.ReadyReplicas != 0 { - t.Error("Failed to set readyReplicas to 0") - } - if set.Status.UpdatedReplicas != 0 { - t.Error("Failed to set updatedReplicas to 0") - } + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + invariants := assertMonotonicInvariants + client := fake.NewSimpleClientset(set) + om, _, ssc, stop := setupController(client) + defer close(stop) + + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up StatefulSet : %s", err) + } + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + *set.Spec.Replicas = 0 + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { + t.Errorf("StatefulSetControl failed to throw error on delete %s", err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn down StatefulSet %s", err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if set.Status.Replicas != 0 { + t.Error("Failed to scale statefulset to 0 replicas") + } + if set.Status.ReadyReplicas != 0 { + t.Error("Failed to set readyReplicas to 0") + } + if set.Status.UpdatedReplicas != 0 { + t.Error("Failed to set updatedReplicas to 0") + } + }) } func TestStatefulSetControl_getSetRevisions(t *testing.T) { @@ -504,9 +612,9 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { testFn := func(test *testcase, t *testing.T) { client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory), &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) - recorder := record.NewFakeRecorder(10) + recorder := &noopRecorder{} ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder} stop := make(chan struct{}) @@ -560,66 +668,70 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { return clone } - set := newStatefulSet(3) - set.Status.CollisionCount = new(int32) - rev0 := newRevisionOrDie(set, 1) - set1 := set.DeepCopy() - set1.Spec.Template.Spec.Containers[0].Image = "foo" - set1.Status.CurrentRevision = rev0.Name - set1.Status.CollisionCount = new(int32) - rev1 := newRevisionOrDie(set1, 2) - set2 := set1.DeepCopy() - set2.Spec.Template.Labels["new"] = "label" - set2.Status.CurrentRevision = rev0.Name - set2.Status.CollisionCount = new(int32) - rev2 := newRevisionOrDie(set2, 3) - tests := []testcase{ - { - name: "creates initial revision", - existing: nil, - set: set, - expectedCount: 1, - expectedCurrent: rev0, - expectedUpdate: rev0, - err: false, - }, - { - name: "creates revision on update", - existing: []*apps.ControllerRevision{rev0}, - set: set1, - expectedCount: 2, - expectedCurrent: rev0, - expectedUpdate: rev1, - err: false, - }, - { - name: "must not recreate a new revision of same set", - existing: []*apps.ControllerRevision{rev0, rev1}, - set: set1, - expectedCount: 2, - expectedCurrent: rev0, - expectedUpdate: rev1, - err: false, - }, - { - name: "must rollback to a previous revision", - existing: []*apps.ControllerRevision{rev0, rev1, rev2}, - set: set1, - expectedCount: 3, - expectedCurrent: rev0, - expectedUpdate: updateRevision(rev1, 4), - err: false, - }, - } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + set.Status.CollisionCount = new(int32) + rev0 := newRevisionOrDie(set, 1) + set1 := set.DeepCopy() + set1.Spec.Template.Spec.Containers[0].Image = "foo" + set1.Status.CurrentRevision = rev0.Name + set1.Status.CollisionCount = new(int32) + rev1 := newRevisionOrDie(set1, 2) + set2 := set1.DeepCopy() + set2.Spec.Template.Labels["new"] = "label" + set2.Status.CurrentRevision = rev0.Name + set2.Status.CollisionCount = new(int32) + rev2 := newRevisionOrDie(set2, 3) + tests := []testcase{ + { + name: "creates initial revision", + existing: nil, + set: set, + expectedCount: 1, + expectedCurrent: rev0, + expectedUpdate: rev0, + err: false, + }, + { + name: "creates revision on update", + existing: []*apps.ControllerRevision{rev0}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must not recreate a new revision of same set", + existing: []*apps.ControllerRevision{rev0, rev1}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must rollback to a previous revision", + existing: []*apps.ControllerRevision{rev0, rev1, rev2}, + set: set1, + expectedCount: 3, + expectedCurrent: rev0, + expectedUpdate: updateRevision(rev1, 4), + err: false, + }, + } + for i := range tests { + testFn(&tests[i], t) + } + }) } func TestStatefulSetControlRollingUpdate(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -628,28 +740,28 @@ func TestStatefulSetControlRollingUpdate(t *testing.T) { testFn := func(test *testcase, t *testing.T) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -792,7 +904,7 @@ func TestStatefulSetControlRollingUpdate(t *testing.T) { func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -801,33 +913,48 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType} client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } + // Pods may have been deleted in the update. Delete any claims with a pod ownerRef. selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + om.claimsIndexer.Delete(claim) + break + } + } + } + + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -838,24 +965,49 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { t.Fatalf("%s: %s", test.name, err) } + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + replicas := *set.Spec.Replicas *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleDownStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } *set.Spec.Replicas = replicas - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1050,23 +1202,26 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { }, }, } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) } func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { type testcase struct { name string partition int32 - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validate func(set *apps.StatefulSet, pods []*v1.Pod) error } - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ Type: apps.RollingUpdateStatefulSetStrategyType, RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { @@ -1074,28 +1229,28 @@ func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { }(), } client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1206,80 +1361,85 @@ func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { }, }, } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) } func TestStatefulSetHonorRevisionHistoryLimit(t *testing.T) { - invariants := assertMonotonicInvariants - set := newStatefulSet(3) - client := fake.NewSimpleClientset(set) - spc, ssu, ssc, stop := setupController(client) - defer close(stop) - - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn up StatefulSet : %s", err) - } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + invariants := assertMonotonicInvariants + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + client := fake.NewSimpleClientset(set) + om, ssu, ssc, stop := setupController(client) + defer close(stop) - for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { - set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) - ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) - updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants) - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up StatefulSet : %s", err) } - revisions, err := ssc.ListRevisions(set) + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { - t.Fatalf("Error listing revisions: %v", err) + t.Fatalf("Error getting updated StatefulSet: %v", err) } - // the extra 2 revisions are `currentRevision` and `updateRevision` - // They're considered as `live`, and truncateHistory only cleans up non-live revisions - if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { - t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) + + for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { + set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) + ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) + updateStatefulSetControl(set, ssc, om, assertUpdateInvariants) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + revisions, err := ssc.ListRevisions(set) + if err != nil { + t.Fatalf("Error listing revisions: %v", err) + } + // the extra 2 revisions are `currentRevision` and `updateRevision` + // They're considered as `live`, and truncateHistory only cleans up non-live revisions + if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { + t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) + } } - } + }) } func TestStatefulSetControlLimitsHistory(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet } - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } for i := 0; i < 10; i++ { set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1314,14 +1474,14 @@ func TestStatefulSetControlLimitsHistory(t *testing.T) { }, } for i := range tests { - testFn(&tests[i], t) + testFn(t, &tests[i]) } } func TestStatefulSetControlRollback(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -1330,31 +1490,31 @@ func TestStatefulSetControlRollback(t *testing.T) { originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1370,17 +1530,17 @@ func TestStatefulSetControlRollback(t *testing.T) { if err != nil { t.Fatalf("%s: %s", test.name, err) } - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1570,7 +1730,7 @@ func TestStatefulSetControlRollback(t *testing.T) { }, } for i := range tests { - testFn(&tests[i], t) + testFn(t, &tests[i]) } } @@ -1659,7 +1819,7 @@ func (rt *requestTracker) reset() { rt.after = 0 } -type fakeStatefulPodControl struct { +type fakeObjectManager struct { podsLister corelisters.PodLister claimsLister corelisters.PersistentVolumeClaimLister setsLister appslisters.StatefulSetLister @@ -1672,14 +1832,18 @@ type fakeStatefulPodControl struct { deletePodTracker requestTracker } -func newFakeStatefulPodControl(podInformer coreinformers.PodInformer, setInformer appsinformers.StatefulSetInformer, revisionInformer appsinformers.ControllerRevisionInformer) *fakeStatefulPodControl { - claimsIndexer := cache.NewIndexer(controller.KeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - return &fakeStatefulPodControl{ +func newFakeObjectManager(informerFactory informers.SharedInformerFactory) *fakeObjectManager { + podInformer := informerFactory.Core().V1().Pods() + claimInformer := informerFactory.Core().V1().PersistentVolumeClaims() + setInformer := informerFactory.Apps().V1().StatefulSets() + revisionInformer := informerFactory.Apps().V1().ControllerRevisions() + + return &fakeObjectManager{ podInformer.Lister(), - corelisters.NewPersistentVolumeClaimLister(claimsIndexer), + claimInformer.Lister(), setInformer.Lister(), podInformer.Informer().GetIndexer(), - claimsIndexer, + claimInformer.Informer().GetIndexer(), setInformer.Informer().GetIndexer(), revisionInformer.Informer().GetIndexer(), requestTracker{0, nil, 0}, @@ -1687,27 +1851,82 @@ func newFakeStatefulPodControl(podInformer coreinformers.PodInformer, setInforme requestTracker{0, nil, 0}} } -func (spc *fakeStatefulPodControl) SetCreateStatefulPodError(err error, after int) { - spc.createPodTracker.err = err - spc.createPodTracker.after = after +func (om *fakeObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error { + defer om.createPodTracker.inc() + if om.createPodTracker.errorReady() { + defer om.createPodTracker.reset() + return om.createPodTracker.err + } + pod.SetUID(types.UID(pod.Name + "-uid")) + return om.podsIndexer.Update(pod) } -func (spc *fakeStatefulPodControl) SetUpdateStatefulPodError(err error, after int) { - spc.updatePodTracker.err = err - spc.updatePodTracker.after = after +func (om *fakeObjectManager) GetPod(namespace, podName string) (*v1.Pod, error) { + return om.podsLister.Pods(namespace).Get(podName) } -func (spc *fakeStatefulPodControl) SetDeleteStatefulPodError(err error, after int) { - spc.deletePodTracker.err = err - spc.deletePodTracker.after = after +func (om *fakeObjectManager) UpdatePod(pod *v1.Pod) error { + return om.podsIndexer.Update(pod) } -func (spc *fakeStatefulPodControl) setPodPending(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) DeletePod(pod *v1.Pod) error { + defer om.deletePodTracker.inc() + if om.deletePodTracker.errorReady() { + defer om.deletePodTracker.reset() + return om.deletePodTracker.err + } + if key, err := controller.KeyFunc(pod); err != nil { + return err + } else if obj, found, err := om.podsIndexer.GetByKey(key); err != nil { + return err + } else if found { + return om.podsIndexer.Delete(obj) + } + return nil // Not found, no error in deleting. +} + +func (om *fakeObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim) error { + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) { + return om.claimsLister.PersistentVolumeClaims(namespace).Get(claimName) +} + +func (om *fakeObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim) error { + // Validate ownerRefs. + refs := claim.GetOwnerReferences() + for _, ref := range refs { + if ref.APIVersion == "" || ref.Kind == "" || ref.Name == "" { + return fmt.Errorf("invalid ownerRefs: %s %v", claim.Name, refs) + } + } + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) SetCreateStatefulPodError(err error, after int) { + om.createPodTracker.err = err + om.createPodTracker.after = after +} + +func (om *fakeObjectManager) SetUpdateStatefulPodError(err error, after int) { + om.updatePodTracker.err = err + om.updatePodTracker.after = after +} + +func (om *fakeObjectManager) SetDeleteStatefulPodError(err error, after int) { + om.deletePodTracker.err = err + om.deletePodTracker.after = after +} + +func (om *fakeObjectManager) setPodPending(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1718,16 +1937,16 @@ func (spc *fakeStatefulPodControl) setPodPending(set *apps.StatefulSet, ordinal pod := pods[ordinal].DeepCopy() pod.Status.Phase = v1.PodPending fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodRunning(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodRunning(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1738,16 +1957,16 @@ func (spc *fakeStatefulPodControl) setPodRunning(set *apps.StatefulSet, ordinal pod := pods[ordinal].DeepCopy() pod.Status.Phase = v1.PodRunning fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodReady(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodReady(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1759,16 +1978,16 @@ func (spc *fakeStatefulPodControl) setPodReady(set *apps.StatefulSet, ordinal in condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} podutil.UpdatePodCondition(&pod.Status, &condition) fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodAvailable(set *apps.StatefulSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodAvailable(set *apps.StatefulSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1792,90 +2011,41 @@ func (spc *fakeStatefulPodControl) setPodAvailable(set *apps.StatefulSet, ordina } podutil.UpdatePodCondition(&pod.Status, &condition) fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) addTerminatingPod(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) addTerminatingPod(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { pod := newStatefulSetPod(set, ordinal) + pod.SetUID(types.UID(pod.Name + "-uid")) // To match fakeObjectManager.CreatePod pod.Status.Phase = v1.PodRunning deleted := metav1.NewTime(time.Now()) pod.DeletionTimestamp = &deleted condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} fakeResourceVersion(pod) podutil.UpdatePodCondition(&pod.Status, &condition) - spc.podsIndexer.Update(pod) + om.podsIndexer.Update(pod) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - return spc.podsLister.Pods(set.Namespace).List(selector) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodTerminated(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodTerminated(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { pod := newStatefulSetPod(set, ordinal) deleted := metav1.NewTime(time.Now()) pod.DeletionTimestamp = &deleted fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) + om.podsIndexer.Update(pod) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - return spc.podsLister.Pods(set.Namespace).List(selector) -} - -func (spc *fakeStatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.createPodTracker.inc() - if spc.createPodTracker.errorReady() { - defer spc.createPodTracker.reset() - return spc.createPodTracker.err - } - - for _, claim := range getPersistentVolumeClaims(set, pod) { - spc.claimsIndexer.Update(&claim) - } - spc.podsIndexer.Update(pod) - return nil -} - -func (spc *fakeStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.updatePodTracker.inc() - if spc.updatePodTracker.errorReady() { - defer spc.updatePodTracker.reset() - return spc.updatePodTracker.err - } - if !identityMatches(set, pod) { - updateIdentity(set, pod) - } - if !storageMatches(set, pod) { - updateStorage(set, pod) - for _, claim := range getPersistentVolumeClaims(set, pod) { - spc.claimsIndexer.Update(&claim) - } - } - spc.podsIndexer.Update(pod) - return nil + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.deletePodTracker.inc() - if spc.deletePodTracker.errorReady() { - defer spc.deletePodTracker.reset() - return spc.deletePodTracker.err - } - if key, err := controller.KeyFunc(pod); err != nil { - return err - } else if obj, found, err := spc.podsIndexer.GetByKey(key); err != nil { - return err - } else if found { - spc.podsIndexer.Delete(obj) - } - - return nil -} - -var _ StatefulPodControlInterface = &fakeStatefulPodControl{} +var _ StatefulPodControlObjectManager = &fakeObjectManager{} type fakeStatefulSetStatusUpdater struct { setsLister appslisters.StatefulSetLister @@ -1909,12 +2079,12 @@ func (ssu *fakeStatefulSetStatusUpdater) SetUpdateStatefulSetStatusError(err err var _ StatefulSetStatusUpdaterInterface = &fakeStatefulSetStatusUpdater{} -func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertMonotonicInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -1933,13 +2103,10 @@ func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodContro } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) - if err != nil { + claim, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) - } } if !identityMatches(set, pods[ord]) { @@ -1949,12 +2116,12 @@ func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodContro return nil } -func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertBurstInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -1965,12 +2132,12 @@ func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) e } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) if err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { + return err } } @@ -1983,12 +2150,12 @@ func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) e return nil } -func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertUpdateInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2000,12 +2167,12 @@ func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) if err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { + return err } } @@ -2031,6 +2198,68 @@ func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) return nil } +func checkClaimInvarients(set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, ordinal int) error { + policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + policy = *set.Spec.PersistentVolumeClaimRetentionPolicy + } + claimShouldBeRetained := policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType + if claim == nil { + if claimShouldBeRetained { + return fmt.Errorf("claim for Pod %s was not created", pod.Name) + } + return nil // A non-retained claim has no invariants to satisfy. + } + + if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(pod) { + // The pod has spun up yet, we do not expect the owner refs on the claim to have been set. + return nil + } + + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for StatefulSet retain", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for StatefulSet retain", claim.Name, pod.Name) + } + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + if !hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s does not have owner ref on %s for StatefulSet deletion", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for StatefulSet deletion", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for scaledown only", claim.Name, set.Name) + } + if ordinal >= int(*set.Spec.Replicas) && !hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s does not have owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + if ordinal < int(*set.Spec.Replicas) && hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + if ordinal >= int(*set.Spec.Replicas) { + if !hasOwnerRef(claim, pod) || hasOwnerRef(claim, set) { + return fmt.Errorf("condemned claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } else { + if hasOwnerRef(claim, pod) || !hasOwnerRef(claim, set) { + return fmt.Errorf("live claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + return nil +} + func fakeResourceVersion(object interface{}) { obj, isObj := object.(metav1.Object) if !isObj { @@ -2045,32 +2274,28 @@ func fakeResourceVersion(object interface{}) { func scaleUpStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, - spc *fakeStatefulPodControl, + om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } for set.Status.ReadyReplicas < *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } sort.Sort(ascendingOrdinal(pods)) // ensure all pods are valid (have a phase) - initialized := false for ord, pod := range pods { if pod.Status.Phase == "" { - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } break } } - if initialized { - continue - } // select one of the pods and move it forward in status if len(pods) > 0 { @@ -2078,11 +2303,11 @@ func scaleUpStatefulSetControl(set *apps.StatefulSet, pod := pods[ord] switch pod.Status.Phase { case v1.PodPending: - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } case v1.PodRunning: - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } default: @@ -2094,25 +2319,26 @@ func scaleUpStatefulSetControl(set *apps.StatefulSet, if err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { return err } //fmt.Printf("Ravig pod conditions %v %v", set.Status.ReadyReplicas, *set.Spec.Replicas) } - return invariants(set, spc) + return invariants(set, om) } -func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, spc *fakeStatefulPodControl, invariants invariantFunc) error { +func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } + for set.Status.Replicas > *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2121,42 +2347,72 @@ func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlIn if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if pods, err = spc.addTerminatingPod(set, ordinal); err != nil { + if pods, err = om.addTerminatingPod(set, ordinal); err != nil { return err } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } sort.Sort(ascendingOrdinal(pods)) if len(pods) > 0 { - spc.podsIndexer.Delete(pods[len(pods)-1]) + om.podsIndexer.Delete(pods[len(pods)-1]) } } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + + if err := invariants(set, om); err != nil { return err } } - return invariants(set, spc) + // If there are claims with ownerRefs on pods that have been deleted, delete them. + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + currentPods := map[string]bool{} + for _, pod := range pods { + currentPods[pod.Name] = true + } + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + return err + } + for _, claim := range claims { + claimPodName := getClaimPodName(set, claim) + if claimPodName == "" { + continue // Skip claims not related to a stateful set pod. + } + if _, found := currentPods[claimPodName]; found { + continue // Skip claims which still have a current pod. + } + for _, refs := range claim.GetOwnerReferences() { + if refs.Name == claimPodName { + om.claimsIndexer.Delete(claim) + break + } + } + } + + return invariants(set, om) } func updateComplete(set *apps.StatefulSet, pods []*v1.Pod) bool { @@ -2198,13 +2454,13 @@ func updateComplete(set *apps.StatefulSet, pods []*v1.Pod) bool { func updateStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, - spc *fakeStatefulPodControl, + om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2212,16 +2468,16 @@ func updateStatefulSetControl(set *apps.StatefulSet, return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } for !updateComplete(set, pods) { - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2229,7 +2485,7 @@ func updateStatefulSetControl(set *apps.StatefulSet, initialized := false for ord, pod := range pods { if pod.Status.Phase == "" { - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } break @@ -2244,11 +2500,11 @@ func updateStatefulSetControl(set *apps.StatefulSet, pod := pods[ord] switch pod.Status.Phase { case v1.PodPending: - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } case v1.PodRunning: - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } default: @@ -2259,19 +2515,19 @@ func updateStatefulSetControl(set *apps.StatefulSet, if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } } - return invariants(set, spc) + return invariants(set, om) } func newRevisionOrDie(set *apps.StatefulSet, revision int64) *apps.ControllerRevision { diff --git a/pkg/controller/statefulset/stateful_set_test.go b/pkg/controller/statefulset/stateful_set_test.go index 31bd9c332058..1f2e98614a0b 100644 --- a/pkg/controller/statefulset/stateful_set_test.go +++ b/pkg/controller/statefulset/stateful_set_test.go @@ -20,21 +20,28 @@ import ( "bytes" "context" "encoding/json" + "fmt" "sort" "testing" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/history" + "k8s.io/kubernetes/pkg/features" ) var parentKind = apps.SchemeGroupVersion.WithKind("StatefulSet") @@ -43,11 +50,11 @@ func alwaysReady() bool { return true } func TestStatefulSetControllerCreates(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -59,11 +66,11 @@ func TestStatefulSetControllerCreates(t *testing.T) { func TestStatefulSetControllerDeletes(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -72,10 +79,10 @@ func TestStatefulSetControllerDeletes(t *testing.T) { t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) } *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn down StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -87,11 +94,11 @@ func TestStatefulSetControllerDeletes(t *testing.T) { func TestStatefulSetControllerRespectsTermination(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -99,11 +106,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { if set.Status.Replicas != 3 { t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) } - _, err := spc.addTerminatingPod(set, 3) + _, err := om.addTerminatingPod(set, 3) if err != nil { t.Error(err) } - pods, err := spc.addTerminatingPod(set, 4) + pods, err := om.addTerminatingPod(set, 4) if err != nil { t.Error(err) } @@ -112,7 +119,7 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { if err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -123,10 +130,10 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { spc.DeleteStatefulPod(set, pods[3]) spc.DeleteStatefulPod(set, pods[4]) *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn down StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -138,11 +145,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { func TestStatefulSetControllerBlocksScaling(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -152,8 +159,8 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { } *set.Spec.Replicas = 5 fakeResourceVersion(set) - spc.setsIndexer.Update(set) - _, err := spc.setPodTerminated(set, 0) + om.setsIndexer.Update(set) + _, err := om.setPodTerminated(set, 0) if err != nil { t.Error("Failed to set pod terminated at ordinal 0") } @@ -163,7 +170,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -174,7 +181,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { spc.DeleteStatefulPod(set, pods[0]) ssc.enqueueStatefulSet(set) fakeWorker(ssc) - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -186,9 +193,9 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { func TestStatefulSetControllerDeletionTimestamp(t *testing.T) { set := newStatefulSet(3) set.DeletionTimestamp = new(metav1.Time) - ssc, spc, _ := newFakeStatefulSetController(set) + ssc, _, om, _ := newFakeStatefulSetController(set) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) // Force a sync. It should not try to create any Pods. ssc.enqueueStatefulSet(set) @@ -198,7 +205,7 @@ func TestStatefulSetControllerDeletionTimestamp(t *testing.T) { if err != nil { t.Fatal(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatal(err) } @@ -211,17 +218,17 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { set := newStatefulSet(3) // The bare client says it IS deleted. set.DeletionTimestamp = new(metav1.Time) - ssc, spc, ssh := newFakeStatefulSetController(set) + ssc, _, om, ssh := newFakeStatefulSetController(set) // The lister (cache) says it's NOT deleted. set2 := *set set2.DeletionTimestamp = nil - spc.setsIndexer.Add(&set2) + om.setsIndexer.Add(&set2) // The recheck occurs in the presence of a matching orphan. pod := newStatefulSetPod(set, 1) pod.OwnerReferences = nil - spc.podsIndexer.Add(pod) + om.podsIndexer.Add(pod) set.Status.CollisionCount = new(int32) revision, err := newRevision(set, 1, set.Status.CollisionCount) if err != nil { @@ -241,7 +248,7 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { if err != nil { t.Fatal(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatal(err) } @@ -272,13 +279,13 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { } func TestStatefulSetControllerAddPod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) ssc.addPod(pod1) key, done := ssc.queue.Get() @@ -304,7 +311,7 @@ func TestStatefulSetControllerAddPod(t *testing.T) { } func TestStatefulSetControllerAddPodOrphan(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" @@ -312,9 +319,9 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) { set3.Name = "foo3" set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"} pod := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) - spc.setsIndexer.Add(set3) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.setsIndexer.Add(set3) // Make pod an orphan. Expect matching sets to be queued. pod.OwnerReferences = nil @@ -325,7 +332,7 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) { } func TestStatefulSetControllerAddPodNoSet(t *testing.T) { - ssc, _, _ := newFakeStatefulSetController() + ssc, _, _, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) ssc.addPod(pod) @@ -337,14 +344,14 @@ func TestStatefulSetControllerAddPodNoSet(t *testing.T) { } func TestStatefulSetControllerUpdatePod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) prev := *pod1 fakeResourceVersion(pod1) @@ -372,7 +379,7 @@ func TestStatefulSetControllerUpdatePod(t *testing.T) { } func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) { - ssc, _, _ := newFakeStatefulSetController() + ssc, _, _, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) prev := *pod @@ -386,10 +393,10 @@ func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) { } func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) ssc.updatePod(pod, pod) ssc.queue.ShutDown() key, _ := ssc.queue.Get() @@ -399,14 +406,14 @@ func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) { } func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) pod.OwnerReferences = nil set2 := newStatefulSet(3) set2.Name = "foo2" - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.Labels = map[string]string{"foo2": "bar2"} fakeResourceVersion(&clone) @@ -417,14 +424,14 @@ func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { } func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.OwnerReferences = pod2.OwnerReferences fakeResourceVersion(&clone) @@ -435,13 +442,13 @@ func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) { } func TestStatefulSetControllerUpdatePodRelease(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.OwnerReferences = nil fakeResourceVersion(&clone) @@ -452,14 +459,14 @@ func TestStatefulSetControllerUpdatePodRelease(t *testing.T) { } func TestStatefulSetControllerDeletePod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) ssc.deletePod(pod1) key, done := ssc.queue.Get() @@ -483,13 +490,13 @@ func TestStatefulSetControllerDeletePod(t *testing.T) { } func TestStatefulSetControllerDeletePodOrphan(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) pod1.OwnerReferences = nil ssc.deletePod(pod1) @@ -499,10 +506,10 @@ func TestStatefulSetControllerDeletePodOrphan(t *testing.T) { } func TestStatefulSetControllerDeletePodTombstone(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) tombstoneKey, _ := controller.KeyFunc(pod) tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod} ssc.deletePod(tombstone) @@ -517,14 +524,14 @@ func TestStatefulSetControllerDeletePodTombstone(t *testing.T) { } func TestStatefulSetControllerGetStatefulSetsForPod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) - spc.podsIndexer.Add(pod) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.podsIndexer.Add(pod) sets := ssc.getStatefulSetsForPod(pod) if got, want := len(sets), 2; got != want { t.Errorf("len(sets) = %v, want %v", got, want) @@ -546,12 +553,12 @@ func TestGetPodsForStatefulSetAdopt(t *testing.T) { pod4.OwnerReferences = nil pod4.Name = "x" + pod4.Name - ssc, spc, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4) + ssc, _, om, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4) - spc.podsIndexer.Add(pod1) - spc.podsIndexer.Add(pod2) - spc.podsIndexer.Add(pod3) - spc.podsIndexer.Add(pod4) + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatal(err) @@ -588,10 +595,10 @@ func TestAdoptOrphanRevisions(t *testing.T) { ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ssc, spc, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2) + ssc, _, om, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2) - spc.revisionsIndexer.Add(ss1Rev1) - spc.revisionsIndexer.Add(ss1Rev2) + om.revisionsIndexer.Add(ss1Rev1) + om.revisionsIndexer.Add(ss1Rev2) err = ssc.adoptOrphanRevisions(context.TODO(), ss1) if err != nil { @@ -615,7 +622,7 @@ func TestAdoptOrphanRevisions(t *testing.T) { func TestGetPodsForStatefulSetRelease(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) + ssc, _, om, _ := newFakeStatefulSetController(set) pod1 := newStatefulSetPod(set, 1) // pod2 is owned but has wrong name. pod2 := newStatefulSetPod(set, 2) @@ -628,9 +635,10 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) { pod4.OwnerReferences = nil pod4.Labels = nil - spc.podsIndexer.Add(pod1) - spc.podsIndexer.Add(pod2) - spc.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatal(err) @@ -651,10 +659,255 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) { } } -func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *fakeStatefulPodControl, history.Interface) { +func TestOrphanedPodsWithPVCDeletePolicy(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + testFn := func(t *testing.T, scaledownPolicy, deletionPolicy apps.PersistentVolumeClaimRetentionPolicyType) { + set := newStatefulSet(4) + *set.Spec.Replicas = 2 + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = scaledownPolicy + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = deletionPolicy + ssc, _, om, _ := newFakeStatefulSetController(set) + om.setsIndexer.Add(set) + + pods := []*v1.Pod{} + pods = append(pods, newStatefulSetPod(set, 0)) + // pod1 is orphaned + pods = append(pods, newStatefulSetPod(set, 1)) + pods[1].OwnerReferences = nil + // pod2 is owned but has wrong name. + pods = append(pods, newStatefulSetPod(set, 2)) + pods[2].Name = "x" + pods[2].Name + + ssc.kubeClient.(*fake.Clientset).PrependReactor("patch", "pods", func(action core.Action) (bool, runtime.Object, error) { + patch := action.(core.PatchAction).GetPatch() + target := action.(core.PatchAction).GetName() + var pod *v1.Pod + for _, p := range pods { + if p.Name == target { + pod = p + break + } + } + if pod == nil { + t.Fatalf("Can't find patch target %s", target) + } + original, err := json.Marshal(pod) + if err != nil { + t.Fatalf("failed to marshal original pod %s: %v", pod.Name, err) + } + updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Pod{}) + if err != nil { + t.Fatalf("failed to apply strategic merge patch %q on node %s: %v", patch, pod.Name, err) + } + if err := json.Unmarshal(updated, pod); err != nil { + t.Fatalf("failed to unmarshal updated pod %s: %v", pod.Name, err) + } + + return true, pod, nil + }) + + for _, pod := range pods { + om.podsIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + om.CreateClaim(&claim) + } + } + + for i := range pods { + if _, err := om.setPodReady(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + if _, err := om.setPodRunning(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + } + + // First sync to manage orphaned pod, then set replicas. + ssc.enqueueStatefulSet(set) + fakeWorker(ssc) + *set.Spec.Replicas = 0 // Put an ownerRef for all scale-down deleted PVCs. + ssc.enqueueStatefulSet(set) + fakeWorker(ssc) + + hasNamedOwnerRef := func(claim *v1.PersistentVolumeClaim, name string) bool { + for _, ownerRef := range claim.GetOwnerReferences() { + if ownerRef.Name == name { + return true + } + } + return false + } + verifyOwnerRefs := func(claim *v1.PersistentVolumeClaim, condemned bool) { + podName := getClaimPodName(set, claim) + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case scaledownPolicy == retain && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == retain && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) != condemned || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) != condemned || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + claims, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(labels.Everything()) + if len(claims) != len(pods) { + t.Errorf("Unexpected number of claims: %d", len(claims)) + } + for _, claim := range claims { + // Only the first pod and the reclaimed orphan pod should have owner refs. + switch claim.Name { + case "datadir-foo-0", "datadir-foo-1": + verifyOwnerRefs(claim, false) + case "datadir-foo-2": + if hasNamedOwnerRef(claim, getClaimPodName(set, claim)) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("unexpected ownerRefs for %s: %v", claim.Name, claim.GetOwnerReferences()) + } + default: + t.Errorf("Unexpected claim %s", claim.Name) + } + } + } + policies := []apps.PersistentVolumeClaimRetentionPolicyType{ + apps.RetainPersistentVolumeClaimRetentionPolicyType, + apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + for _, scaledownPolicy := range policies { + for _, deletionPolicy := range policies { + testName := fmt.Sprintf("ScaleDown:%s/SetDeletion:%s", scaledownPolicy, deletionPolicy) + t.Run(testName, func(t *testing.T) { testFn(t, scaledownPolicy, deletionPolicy) }) + } + } +} + +func TestStaleOwnerRefOnScaleup(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } { + onPolicy := func(msg string, args ...interface{}) string { + return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...) + } + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to turn up StatefulSet : %s", err)) + } + var err error + if set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name); err != nil { + t.Errorf(onPolicy("Could not get scaled up set: %v", err)) + } + if set.Status.Replicas != 3 { + t.Errorf(onPolicy("set.Status.Replicas = %v; want 3", set.Status.Replicas)) + } + *set.Spec.Replicas = 2 + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to scale down StatefulSet : msg, %s", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Errorf(onPolicy("Failed to scale statefulset to 2 replicas")) + } + + var claim *v1.PersistentVolumeClaim + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs := claim.GetOwnerReferences() + if len(refs) != 1 { + t.Errorf(onPolicy("Expected only one refs: %v", refs)) + } + // Make the pod ref stale. + for i := range refs { + if refs[i].Name == "foo-2" { + refs[i].UID = "stale" + break + } + } + claim.SetOwnerReferences(refs) + if err = om.claimsIndexer.Update(claim); err != nil { + t.Errorf(onPolicy("Could not update claim with new owner ref: %v", err)) + } + + *set.Spec.Replicas = 3 + // Until the stale PVC goes away, the scale up should never finish. Run 10 iterations, then delete the PVC. + if err := scaleUpStatefulSetControllerBounded(set, ssc, spc, om, 10); err != nil { + t.Errorf(onPolicy("Failed attempt to scale StatefulSet back up: %v", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Errorf(onPolicy("Expected set to stay at two replicas")) + } + + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs = claim.GetOwnerReferences() + if len(refs) != 1 { + t.Errorf(onPolicy("Unexpected change to condemned pvc ownerRefs: %v", refs)) + } + foundPodRef := false + for i := range refs { + if refs[i].UID == "stale" { + foundPodRef = true + break + } + } + if !foundPodRef { + t.Errorf(onPolicy("Claim ref unexpectedly changed: %v", refs)) + } + if err = om.claimsIndexer.Delete(claim); err != nil { + t.Errorf(onPolicy("Could not delete stale pvc: %v", err)) + } + + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to scale StatefulSet back up: %v", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 3 { + t.Errorf(onPolicy("Failed to scale set back up once PVC was deleted")) + } + } +} + +func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *StatefulPodControl, *fakeObjectManager, history.Interface) { client := fake.NewSimpleClientset(initialObjects...) informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - fpc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + om := newFakeObjectManager(informerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) ssc := NewStatefulSetController( informerFactory.Core().V1().Pods(), @@ -667,9 +920,9 @@ func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSe ssc.podListerSynced = alwaysReady ssc.setListerSynced = alwaysReady recorder := record.NewFakeRecorder(10) - ssc.control = NewDefaultStatefulSetControl(fpc, ssu, ssh, recorder) + ssc.control = NewDefaultStatefulSetControl(spc, ssu, ssh, recorder) - return ssc, fpc, ssh + return ssc, spc, om, ssh } func fakeWorker(ssc *StatefulSetController) { @@ -687,21 +940,27 @@ func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { return pods[ordinal] } -func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { - spc.setsIndexer.Add(set) +func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error { + return scaleUpStatefulSetControllerBounded(set, ssc, spc, om, -1) +} + +func scaleUpStatefulSetControllerBounded(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager, maxIterations int) error { + om.setsIndexer.Add(set) ssc.enqueueStatefulSet(set) fakeWorker(ssc) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - for set.Status.ReadyReplicas < *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + iterations := 0 + for (maxIterations < 0 || iterations < maxIterations) && set.Status.ReadyReplicas < *set.Spec.Replicas { + iterations++ + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } ord := len(pods) - 1 - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } pod := getPodAtOrdinal(pods, ord) @@ -709,7 +968,7 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl fakeWorker(ssc) pod = getPodAtOrdinal(pods, ord) prev := *pod - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } pod = getPodAtOrdinal(pods, ord) @@ -717,31 +976,31 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl fakeWorker(ssc) pod = getPodAtOrdinal(pods, ord) prev = *pod - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } pod = getPodAtOrdinal(pods, ord) ssc.updatePod(&prev, pod) fakeWorker(ssc) - if err := assertMonotonicInvariants(set, spc); err != nil { + if err := assertMonotonicInvariants(set, om); err != nil { return err } - obj, _, err := spc.setsIndexer.Get(set) + obj, _, err := om.setsIndexer.Get(set) if err != nil { return err } set = obj.(*apps.StatefulSet) } - return assertMonotonicInvariants(set, spc) + return assertMonotonicInvariants(set, om) } -func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { +func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -749,10 +1008,10 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr pod := getPodAtOrdinal(pods, ord) prev := *pod fakeResourceVersion(set) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) ssc.enqueueStatefulSet(set) fakeWorker(ssc) - pods, err = spc.addTerminatingPod(set, ord) + pods, err = om.addTerminatingPod(set, ord) if err != nil { return err } @@ -763,13 +1022,13 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr ssc.deletePod(pod) fakeWorker(ssc) for set.Status.Replicas > *set.Spec.Replicas { - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } ord := len(pods) - pods, err = spc.addTerminatingPod(set, ord) + pods, err = om.addTerminatingPod(set, ord) if err != nil { return err } @@ -779,14 +1038,14 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr spc.DeleteStatefulPod(set, pod) ssc.deletePod(pod) fakeWorker(ssc) - obj, _, err := spc.setsIndexer.Get(set) + obj, _, err := om.setsIndexer.Get(set) if err != nil { return err } set = obj.(*apps.StatefulSet) } - return assertMonotonicInvariants(set, spc) + return assertMonotonicInvariants(set, om) } func rawTemplate(template *v1.PodTemplateSpec) runtime.RawExtension { diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index 88ae93d2834b..3233a9e5255c 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/history" @@ -132,6 +133,179 @@ func storageMatches(set *apps.StatefulSet, pod *v1.Pod) bool { return true } +// getPersistentVolumeClaimPolicy returns the PVC policy for a StatefulSet, returning a retain policy if the set policy is nil. +func getPersistentVolumeClaimRetentionPolicy(set *apps.StatefulSet) apps.StatefulSetPersistentVolumeClaimRetentionPolicy { + policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if set.Spec.PersistentVolumeClaimRetentionPolicy != nil { + policy = *set.Spec.PersistentVolumeClaimRetentionPolicy + } + return policy +} + +// claimOwnerMatchesSetAndPod returns false if the ownerRefs of the claim are not set consistently with the +// PVC deletion policy for the StatefulSet. +func claimOwnerMatchesSetAndPod(claim *v1.PersistentVolumeClaim, set *apps.StatefulSet, pod *v1.Pod) bool { + policy := set.Spec.PersistentVolumeClaimRetentionPolicy + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + default: + klog.Errorf("Unknown policy %v; treating as Retain", set.Spec.PersistentVolumeClaimRetentionPolicy) + fallthrough + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) || + hasOwnerRef(claim, pod) { + return false + } + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + if !hasOwnerRef(claim, set) || + hasOwnerRef(claim, pod) { + return false + } + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return false + } + podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas) + if podScaledDown != hasOwnerRef(claim, pod) { + return false + } + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas) + // If a pod is scaled down, there should be no set ref and a pod ref; + // if the pod is not scaled down it's the other way around. + if podScaledDown == hasOwnerRef(claim, set) { + return false + } + if podScaledDown != hasOwnerRef(claim, pod) { + return false + } + } + return true +} + +// updateClaimOwnerRefForSetAndPod updates the ownerRefs for the claim according to the deletion policy of +// the StatefulSet. Returns true if the claim was changed and should be updated and false otherwise. +func updateClaimOwnerRefForSetAndPod(claim *v1.PersistentVolumeClaim, set *apps.StatefulSet, pod *v1.Pod) bool { + needsUpdate := false + // Sometimes the version and kind are not set {pod,set}.TypeMeta. These are necessary for the ownerRef. + // This is the case both in real clusters and the unittests. + // TODO: there must be a better way to do this other than hardcoding the pod version? + updateMeta := func(tm *metav1.TypeMeta, kind string) { + if tm.APIVersion == "" { + if kind == "StatefulSet" { + tm.APIVersion = "apps/v1" + } else { + tm.APIVersion = "v1" + } + } + if tm.Kind == "" { + tm.Kind = kind + } + } + podMeta := pod.TypeMeta + updateMeta(&podMeta, "Pod") + setMeta := set.TypeMeta + updateMeta(&setMeta, "StatefulSet") + policy := set.Spec.PersistentVolumeClaimRetentionPolicy + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + default: + klog.Errorf("Unknown policy %v, treating as Retain", set.Spec.PersistentVolumeClaimRetentionPolicy) + fallthrough + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + needsUpdate = removeOwnerRef(claim, set) || needsUpdate + needsUpdate = removeOwnerRef(claim, pod) || needsUpdate + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + needsUpdate = setOwnerRef(claim, set, &setMeta) || needsUpdate + needsUpdate = removeOwnerRef(claim, pod) || needsUpdate + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + needsUpdate = removeOwnerRef(claim, set) || needsUpdate + podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas) + if podScaledDown { + needsUpdate = setOwnerRef(claim, pod, &podMeta) || needsUpdate + } + if !podScaledDown { + needsUpdate = removeOwnerRef(claim, pod) || needsUpdate + } + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas) + if podScaledDown { + needsUpdate = removeOwnerRef(claim, set) || needsUpdate + needsUpdate = setOwnerRef(claim, pod, &podMeta) || needsUpdate + } + if !podScaledDown { + needsUpdate = setOwnerRef(claim, set, &setMeta) || needsUpdate + needsUpdate = removeOwnerRef(claim, pod) || needsUpdate + } + } + return needsUpdate +} + +// hasOwnerRef returns true if target has an ownerRef to owner. +func hasOwnerRef(target, owner metav1.Object) bool { + ownerUID := owner.GetUID() + for _, ownerRef := range target.GetOwnerReferences() { + if ownerRef.UID == ownerUID { + return true + } + } + return false +} + +// hasStaleOwnerRef returns true if target has a ref to owner that appears to be stale. +func hasStaleOwnerRef(target, owner metav1.Object) bool { + for _, ownerRef := range target.GetOwnerReferences() { + if ownerRef.Name == owner.GetName() && ownerRef.UID != owner.GetUID() { + return true + } + } + return false +} + +// setOwnerRef adds owner to the ownerRefs of target, if necessary. Returns true if target needs to be +// updated and false otherwise. +func setOwnerRef(target, owner metav1.Object, ownerType *metav1.TypeMeta) bool { + if hasOwnerRef(target, owner) { + return false + } + ownerRefs := append( + target.GetOwnerReferences(), + metav1.OwnerReference{ + APIVersion: ownerType.APIVersion, + Kind: ownerType.Kind, + Name: owner.GetName(), + UID: owner.GetUID(), + }) + target.SetOwnerReferences(ownerRefs) + return true +} + +// removeOwnerRef removes owner from the ownerRefs of target, if necessary. Returns true if target needs +// to be updated and false otherwise. +func removeOwnerRef(target, owner metav1.Object) bool { + if !hasOwnerRef(target, owner) { + return false + } + ownerUID := owner.GetUID() + oldRefs := target.GetOwnerReferences() + newRefs := make([]metav1.OwnerReference, len(oldRefs)-1) + skip := 0 + for i := range oldRefs { + if oldRefs[i].UID == ownerUID { + skip = -1 + } else { + newRefs[i+skip] = oldRefs[i] + } + } + target.SetOwnerReferences(newRefs) + return true +} + // getPersistentVolumeClaims gets a map of PersistentVolumeClaims to their template names, as defined in set. The // returned PersistentVolumeClaims are each constructed with a the name specific to the Pod. This name is determined // by getPersistentVolumeClaimName. @@ -278,7 +452,10 @@ func newVersionedStatefulSetPod(currentSet, updateSet *apps.StatefulSet, current // Match check if the given StatefulSet's template matches the template stored in the given history. func Match(ss *apps.StatefulSet, history *apps.ControllerRevision) (bool, error) { - patch, err := getPatch(ss) + // Encoding the set for the patch may update its GVK metadata, which causes data races if this + // set is in an informer cache. + clone := ss.DeepCopy() + patch, err := getPatch(clone) if err != nil { return false, err } diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index 1b9f4693e6da..f86dbd3eca2b 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/rand" "reflect" + "regexp" "sort" "strconv" "testing" @@ -27,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" apps "k8s.io/api/apps/v1" @@ -35,6 +37,28 @@ import ( "k8s.io/kubernetes/pkg/controller/history" ) +// noopRecorder is an EventRecorder that does nothing. record.FakeRecorder has a fixed +// buffer size, which causes tests to hang if that buffer's exceeded. +type noopRecorder struct{} + +func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {} +func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} +func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching. +func getClaimPodName(set *apps.StatefulSet, claim *v1.PersistentVolumeClaim) string { + podName := "" + + statefulClaimRegex := regexp.MustCompile(fmt.Sprintf(".*-(%s-[0-9]+)$", set.Name)) + matches := statefulClaimRegex.FindStringSubmatch(claim.Name) + if len(matches) != 2 { + return podName + } + return matches[1] +} + func TestGetParentNameAndOrdinal(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 1) @@ -51,6 +75,28 @@ func TestGetParentNameAndOrdinal(t *testing.T) { } } +func TestGetClaimPodName(t *testing.T) { + set := apps.StatefulSet{} + set.Name = "my-set" + claim := v1.PersistentVolumeClaim{} + claim.Name = "volume-my-set-2" + if pod := getClaimPodName(&set, &claim); pod != "my-set-2" { + t.Errorf("Expected my-set-2 found %s", pod) + } + claim.Name = "long-volume-my-set-20" + if pod := getClaimPodName(&set, &claim); pod != "my-set-20" { + t.Errorf("Expected my-set-20 found %s", pod) + } + claim.Name = "volume-2-my-set" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } + claim.Name = "volume-pod-2" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } +} + func TestIsMemberOf(t *testing.T) { set := newStatefulSet(3) set2 := newStatefulSet(3) @@ -180,6 +226,350 @@ func TestUpdateStorage(t *testing.T) { } } +func TestGetPersistentVolumeClaimRetentionPolicy(t *testing.T) { + retainPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + scaledownPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + + set := apps.StatefulSet{} + set.Spec.PersistentVolumeClaimRetentionPolicy = &retainPolicy + got := getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.RetainPersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected retain policy") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &scaledownPolicy + got = getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected scaledown policy") + } +} + +func TestClaimOwnerMatchesSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + needsPodRef bool + needsSetRef bool + replicas int32 + ordinal int + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "on SS delete", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "on scaledown only, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on scaledown only, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + replicas: 2, + ordinal: 1, + }, + { + name: "on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on both, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + replicas: 2, + ordinal: 1, + }, + } + + for _, tc := range testCases { + for _, useOtherRefs := range []bool{false, true} { + for _, setPodRef := range []bool{false, true} { + for _, setSetRef := range []bool{false, true} { + claim := v1.PersistentVolumeClaim{} + claim.Name = "target-claim" + pod := v1.Pod{} + pod.Name = fmt.Sprintf("pod-%d", tc.ordinal) + pod.GetObjectMeta().SetUID("pod-123") + set := apps.StatefulSet{} + set.Name = "stateful-set" + set.GetObjectMeta().SetUID("ss-456") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + set.Spec.Replicas = &tc.replicas + if setPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if setSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + if useOtherRefs { + randomObject1 := v1.Pod{} + randomObject1.Name = "rand1" + randomObject1.GetObjectMeta().SetUID("rand1-abc") + randomObject2 := v1.Pod{} + randomObject2.Name = "rand2" + randomObject2.GetObjectMeta().SetUID("rand2-def") + setOwnerRef(&claim, &randomObject1, &randomObject1.TypeMeta) + setOwnerRef(&claim, &randomObject2, &randomObject2.TypeMeta) + } + shouldMatch := setPodRef == tc.needsPodRef && setSetRef == tc.needsSetRef + if claimOwnerMatchesSetAndPod(&claim, &set, &pod) != shouldMatch { + t.Errorf("Bad match for %s with pod=%v,set=%v,others=%v", tc.name, setPodRef, setSetRef, useOtherRefs) + } + } + } + } + } +} + +func TestUpdateClaimOwnerRefForSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + condemned bool + needsPodRef bool + needsSetRef bool + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete with set", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete with scaledown, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete on scaledown, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + { + name: "delete on both, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + } + for _, tc := range testCases { + for _, hasPodRef := range []bool{true, false} { + for _, hasSetRef := range []bool{true, false} { + set := apps.StatefulSet{} + set.Name = "ss" + numReplicas := int32(5) + set.Spec.Replicas = &numReplicas + set.SetUID("ss-123") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + pod := v1.Pod{} + if tc.condemned { + pod.Name = "pod-8" + } else { + pod.Name = "pod-1" + } + pod.SetUID("pod-456") + claim := v1.PersistentVolumeClaim{} + if hasPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if hasSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + needsUpdate := hasPodRef != tc.needsPodRef || hasSetRef != tc.needsSetRef + shouldUpdate := updateClaimOwnerRefForSetAndPod(&claim, &set, &pod) + if shouldUpdate != needsUpdate { + t.Errorf("Bad update for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &pod) != tc.needsPodRef { + t.Errorf("Bad pod ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &set) != tc.needsSetRef { + t.Errorf("Bad set ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + } + } + } +} + +func TestHasOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {UID: "123"}, {UID: "456"}}) + ownerA := v1.Pod{} + ownerA.GetObjectMeta().SetUID("123") + ownerB := v1.Pod{} + ownerB.GetObjectMeta().SetUID("789") + if !hasOwnerRef(&target, &ownerA) { + t.Error("Missing owner") + } + if hasOwnerRef(&target, &ownerB) { + t.Error("Unexpected owner") + } +} + +func TestHasStaleOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "bob", UID: "123"}, {Name: "shirley", UID: "456"}}) + ownerA := v1.Pod{} + ownerA.SetUID("123") + ownerA.Name = "bob" + ownerB := v1.Pod{} + ownerB.Name = "shirley" + ownerB.SetUID("789") + ownerC := v1.Pod{} + ownerC.Name = "yvonne" + ownerC.SetUID("345") + if hasStaleOwnerRef(&target, &ownerA) { + t.Error("ownerA should not be stale") + } + if !hasStaleOwnerRef(&target, &ownerB) { + t.Error("ownerB should be stale") + } + if hasStaleOwnerRef(&target, &ownerC) { + t.Error("ownerC should not be stale") + } +} + +func TestSetOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != true { + t.Errorf("Unexpected lack of update") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" { + t.Errorf("Unexpected owner UID %v", ownerRefs[0].UID) + } + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != false { + t.Errorf("Unexpected update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Unexpected duplicate reference") + } + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + if setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) != true { + t.Error("Unexpected lack of second update") + } + ownerRefs = target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 2 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" || ownerRefs[1].UID != "BCD" { + t.Errorf("Bad second ownerRefs: %v", ownerRefs) + } +} + +func TestRemoveOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if removeOwnerRef(&target, &ownerA) != false { + t.Error("Unexpected update on empty remove") + } + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Unexpected lack of update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 0 { + t.Error("Unexpected owner reference remains") + } + + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerB) != false { + t.Error("Unexpected update for mismatched owner") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Missing ref after no-op remove") + } + setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Missing update for second remove") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Error("Extra ref after second remove") + } + if ownerRefs[0].UID != "BCD" { + t.Error("Bad UID after second remove") + } +} + func TestIsRunningAndReady(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 1) @@ -387,7 +777,8 @@ func newPod() *v1.Pod { func newPVC(name string) v1.PersistentVolumeClaim { return v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Namespace: "default", + Name: name, }, Spec: v1.PersistentVolumeClaimSpec{ Resources: v1.ResourceRequirements{ @@ -452,6 +843,10 @@ func newStatefulSetWithVolumes(replicas int, name string, petMounts []v1.VolumeM VolumeClaimTemplates: claims, ServiceName: "governingsvc", UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, RevisionHistoryLimit: func() *int32 { limit := int32(2) return &limit @@ -499,6 +894,10 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m MatchExpressions: testMatchExpressions, }, Replicas: func() *int32 { i := int32(replicas); return &i }(), + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -525,7 +924,7 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m }, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ { - ObjectMeta: metav1.ObjectMeta{Name: "datadir"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "datadir"}, Spec: v1.PersistentVolumeClaimSpec{ Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 88cc6bb057e5..7d3a89d90c9d 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -610,6 +610,12 @@ const ( // alpha: v1.21 VolumeCapacityPriority featuregate.Feature = "VolumeCapacityPriority" + // owner: @mattcary + // alpha: v1.22 + // + // Enables policies controlling deletion of PVCs created by a StatefulSet. + StatefulSetAutoDeletePVC featuregate.Feature = "StatefulSetAutoDeletePVC" + // owner: @ahg-g // alpha: v1.21 // beta: v1.22 @@ -941,6 +947,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ProbeTerminationGracePeriod: {Default: false, PreRelease: featuregate.Beta}, // Default to false in beta 1.22, set to true in 1.24 NodeSwap: {Default: false, PreRelease: featuregate.Alpha}, PodDeletionCost: {Default: true, PreRelease: featuregate.Beta}, + StatefulSetAutoDeletePVC: {Default: false, PreRelease: featuregate.Alpha}, TopologyAwareHints: {Default: false, PreRelease: featuregate.Beta}, PodAffinityNamespaceSelector: {Default: true, PreRelease: featuregate.Beta}, ServiceLoadBalancerClass: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/registry/apps/statefulset/strategy.go b/pkg/registry/apps/statefulset/strategy.go index 21ed15771d86..eec69049269b 100644 --- a/pkg/registry/apps/statefulset/strategy.go +++ b/pkg/registry/apps/statefulset/strategy.go @@ -18,18 +18,18 @@ package statefulset import ( "context" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/features" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/pod" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/apps/validation" + "k8s.io/kubernetes/pkg/features" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) @@ -74,6 +74,7 @@ func (statefulSetStrategy) PrepareForCreate(ctx context.Context, obj runtime.Obj statefulSet.Status = apps.StatefulSetStatus{} statefulSet.Generation = 1 + dropStatefulSetDisabledFields(statefulSet, nil) pod.DropDisabledTemplateFields(&statefulSet.Spec.Template, nil) } @@ -108,6 +109,12 @@ func dropStatefulSetDisabledFields(newSS *apps.StatefulSet, oldSS *apps.Stateful newSS.Spec.MinReadySeconds = int32(0) } } + + if !utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if oldSS == nil || oldSS.Spec.PersistentVolumeClaimRetentionPolicy == nil { + newSS.Spec.PersistentVolumeClaimRetentionPolicy = nil + } + } } // minReadySecondsFieldsInUse returns true if fields related to StatefulSet minReadySeconds are set and diff --git a/pkg/registry/apps/statefulset/strategy_test.go b/pkg/registry/apps/statefulset/strategy_test.go index 1b360cea5d30..757f6f59a1bb 100644 --- a/pkg/registry/apps/statefulset/strategy_test.go +++ b/pkg/registry/apps/statefulset/strategy_test.go @@ -178,13 +178,106 @@ func TestStatefulSetStrategy(t *testing.T) { if len(errs) == 0 { t.Errorf("updating only spec.Replicas is allowed on a statefulset: %v", errs) } - expectedUpdateErrorString := "spec: Forbidden: updates to statefulset spec for fields other than 'replicas'," + - " 'template', 'updateStrategy' and 'minReadySeconds' are forbidden" + expectedUpdateErrorString := "spec: Forbidden: updates to statefulset spec for fields other than" + + " 'replicas', 'template', 'updateStrategy' and 'persistentVolumeClaimRetentionPolicy' are forbidden" if errs[0].Error() != expectedUpdateErrorString { t.Errorf("expected error string %v", errs[0].Error()) } }) + validPs = &apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: ps.Name, Namespace: ps.Namespace, ResourceVersion: "1", Generation: 1}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: ps.Spec.Selector, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + Status: apps.StatefulSetStatus{Replicas: 4}, + } + + t.Run("when StatefulSetAutoDeletePVC feature gate is enabled, PersistentVolumeClaimRetentionPolicy should be updated", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + // Test creation + ps := &apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: ps.Spec.Selector, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.PersistentVolumeClaimRetentionPolicyType("invalid policy"), + }, + }, + } + Strategy.PrepareForCreate(ctx, ps) + errs := Strategy.Validate(ctx, ps) + if len(errs) == 0 { + t.Errorf("expected failure when PersistentVolumeClaimRetentionPolicy is invalid") + } + expectedCreateErrorString := "spec.persistentVolumeClaimRetentionPolicy.whenDeleted: Unsupported value: \"invalid policy\": supported values: \"Retain\", \"Delete\"" + if errs[0].Error() != expectedCreateErrorString { + t.Errorf("mismatched error string %v (expected %v)", errs[0].Error(), expectedCreateErrorString) + } + Strategy.PrepareForUpdate(ctx, validPs, ps) + errs = Strategy.ValidateUpdate(ctx, validPs, ps) + if len(errs) != 0 { + t.Errorf("updates to PersistentVolumeClaimRetentionPolicy should be allowed: %v", errs) + } + invalidPs := ps + invalidPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType("invalid type") + Strategy.PrepareForUpdate(ctx, validPs, invalidPs) + errs = Strategy.ValidateUpdate(ctx, validPs, ps) + if len(errs) != 0 { + t.Errorf("invalid updates to PersistentVolumeClaimRetentionPolicyType should be allowed: %v", errs) + } + if validPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType || validPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType { + t.Errorf("expected PersistentVolumeClaimRetentionPolicy to be updated: %v", errs) + } + }) + t.Run("when StatefulSetAutoDeletePVC feature gate is disabled, PersistentVolumeClaimRetentionPolicy should not be updated", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + // Test creation + ps := &apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: ps.Spec.Selector, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + } + Strategy.PrepareForCreate(ctx, ps) + errs := Strategy.Validate(ctx, ps) + if len(errs) != 0 { + t.Errorf("unexpected failure with PersistentVolumeClaimRetentionPolicy: %v", errs) + } + if ps.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType || ps.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType { + t.Errorf("expected invalid PersistentVolumeClaimRetentionPolicy to be defaulted to Retain, but got %v", ps.Spec.PersistentVolumeClaimRetentionPolicy) + } + Strategy.PrepareForUpdate(ctx, validPs, ps) + errs = Strategy.ValidateUpdate(ctx, validPs, ps) + if len(errs) != 0 { + t.Errorf("updates to PersistentVolumeClaimRetentionPolicy should be allowed: %v", errs) + } + invalidPs := ps + invalidPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType("invalid type") + Strategy.PrepareForUpdate(ctx, validPs, invalidPs) + errs = Strategy.ValidateUpdate(ctx, validPs, ps) + if len(errs) != 0 { + t.Errorf("should ignore updates to PersistentVolumeClaimRetentionPolicyType") + } + }) + validPs.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"a": "bar"}} Strategy.PrepareForUpdate(ctx, validPs, ps) errs = Strategy.ValidateUpdate(ctx, validPs, ps) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index 5ff47db220a3..2b74e848953f 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -339,19 +339,27 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) eventsRule(), }, }) - addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "statefulset-controller"}, - Rules: []rbacv1.PolicyRule{ - rbacv1helpers.NewRule("list", "watch").Groups(legacyGroup).Resources("pods").RuleOrDie(), - rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), - rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/status").RuleOrDie(), - rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/finalizers").RuleOrDie(), - rbacv1helpers.NewRule("get", "create", "delete", "update", "patch").Groups(legacyGroup).Resources("pods").RuleOrDie(), - rbacv1helpers.NewRule("get", "create", "delete", "update", "patch", "list", "watch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(), - rbacv1helpers.NewRule("get", "create").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(), - eventsRule(), - }, - }) + addControllerRole(&controllerRoles, &controllerRoleBindings, func() rbacv1.ClusterRole { + role := rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "statefulset-controller"}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("list", "watch").Groups(legacyGroup).Resources("pods").RuleOrDie(), + rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), + rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/status").RuleOrDie(), + rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/finalizers").RuleOrDie(), + rbacv1helpers.NewRule("get", "create", "delete", "update", "patch").Groups(legacyGroup).Resources("pods").RuleOrDie(), + rbacv1helpers.NewRule("get", "create", "delete", "update", "patch", "list", "watch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(), + rbacv1helpers.NewRule("get", "create").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(), + eventsRule(), + }, + } + + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + role.Rules = append(role.Rules, rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie()) + } + + return role + }()) addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "ttl-controller"}, Rules: []rbacv1.PolicyRule{ diff --git a/staging/src/k8s.io/api/apps/v1/generated.pb.go b/staging/src/k8s.io/api/apps/v1/generated.pb.go index 0a15aff4d8c2..81d51bd58f5c 100644 --- a/staging/src/k8s.io/api/apps/v1/generated.pb.go +++ b/staging/src/k8s.io/api/apps/v1/generated.pb.go @@ -748,10 +748,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() { var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() { + *m = StatefulSetPersistentVolumeClaimRetentionPolicy{} +} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) { + return fileDescriptor_e1014cab6f31e43b, []int{25} +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int { + return m.Size() +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m) +} + +var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo + func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_e1014cab6f31e43b, []int{25} + return fileDescriptor_e1014cab6f31e43b, []int{26} } func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -779,7 +809,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_e1014cab6f31e43b, []int{26} + return fileDescriptor_e1014cab6f31e43b, []int{27} } func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -807,7 +837,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_e1014cab6f31e43b, []int{27} + return fileDescriptor_e1014cab6f31e43b, []int{28} } func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -858,6 +888,7 @@ func init() { proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1.StatefulSet") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1.StatefulSetCondition") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1.StatefulSetList") + proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1.StatefulSetSpec") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1.StatefulSetStatus") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1.StatefulSetUpdateStrategy") @@ -868,135 +899,142 @@ func init() { } var fileDescriptor_e1014cab6f31e43b = []byte{ - // 2047 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x24, 0x47, - 0x15, 0x77, 0xcf, 0x87, 0x3d, 0x2e, 0xaf, 0xed, 0xdd, 0xb2, 0xb1, 0x27, 0xbb, 0x64, 0x66, 0x19, - 0x60, 0xe3, 0x64, 0xb3, 0x3d, 0xec, 0x66, 0x13, 0xa1, 0x2c, 0x02, 0x79, 0xc6, 0x21, 0x84, 0x78, - 0x6c, 0x53, 0x5e, 0xef, 0x61, 0x09, 0x12, 0xe5, 0xe9, 0xda, 0x71, 0xc7, 0xfd, 0xa5, 0xee, 0xea, - 0x61, 0x47, 0x5c, 0x10, 0x12, 0x27, 0x38, 0xf0, 0x9f, 0x20, 0x84, 0xc8, 0x0d, 0x45, 0x88, 0xcb, - 0x5e, 0x90, 0x22, 0x2e, 0xe4, 0x64, 0xb1, 0x93, 0x13, 0x42, 0x1c, 0xb9, 0xe4, 0x02, 0xaa, 0xea, - 0xea, 0xef, 0x6a, 0xcf, 0xd8, 0x9b, 0x38, 0x24, 0xca, 0x6d, 0xba, 0xea, 0xf7, 0x7e, 0xf5, 0x5e, - 0xd5, 0xab, 0x7a, 0xbf, 0xae, 0x1e, 0x70, 0xef, 0xf8, 0xdb, 0x9e, 0xaa, 0xdb, 0xed, 0x63, 0xff, - 0x90, 0xb8, 0x16, 0xa1, 0xc4, 0x6b, 0x0f, 0x89, 0xa5, 0xd9, 0x6e, 0x5b, 0x74, 0x60, 0x47, 0x6f, - 0x63, 0xc7, 0xf1, 0xda, 0xc3, 0xdb, 0xed, 0x01, 0xb1, 0x88, 0x8b, 0x29, 0xd1, 0x54, 0xc7, 0xb5, - 0xa9, 0x0d, 0x61, 0x80, 0x51, 0xb1, 0xa3, 0xab, 0x0c, 0xa3, 0x0e, 0x6f, 0x5f, 0xbd, 0x35, 0xd0, - 0xe9, 0x91, 0x7f, 0xa8, 0xf6, 0x6d, 0xb3, 0x3d, 0xb0, 0x07, 0x76, 0x9b, 0x43, 0x0f, 0xfd, 0x47, - 0xfc, 0x89, 0x3f, 0xf0, 0x5f, 0x01, 0xc5, 0xd5, 0x56, 0x62, 0x98, 0xbe, 0xed, 0x12, 0xc9, 0x30, - 0x57, 0xef, 0xc6, 0x18, 0x13, 0xf7, 0x8f, 0x74, 0x8b, 0xb8, 0xa3, 0xb6, 0x73, 0x3c, 0x60, 0x0d, - 0x5e, 0xdb, 0x24, 0x14, 0xcb, 0xac, 0xda, 0x45, 0x56, 0xae, 0x6f, 0x51, 0xdd, 0x24, 0x39, 0x83, - 0xd7, 0x26, 0x19, 0x78, 0xfd, 0x23, 0x62, 0xe2, 0x9c, 0xdd, 0x2b, 0x45, 0x76, 0x3e, 0xd5, 0x8d, - 0xb6, 0x6e, 0x51, 0x8f, 0xba, 0x59, 0xa3, 0xd6, 0x7f, 0x14, 0x00, 0xbb, 0xb6, 0x45, 0x5d, 0xdb, - 0x30, 0x88, 0x8b, 0xc8, 0x50, 0xf7, 0x74, 0xdb, 0x82, 0x3f, 0x05, 0x35, 0x16, 0x8f, 0x86, 0x29, - 0xae, 0x2b, 0xd7, 0x95, 0x8d, 0x85, 0x3b, 0xdf, 0x52, 0xe3, 0x49, 0x8e, 0xe8, 0x55, 0xe7, 0x78, - 0xc0, 0x1a, 0x3c, 0x95, 0xa1, 0xd5, 0xe1, 0x6d, 0x75, 0xf7, 0xf0, 0x5d, 0xd2, 0xa7, 0x3d, 0x42, - 0x71, 0x07, 0x3e, 0x39, 0x69, 0xce, 0x8c, 0x4f, 0x9a, 0x20, 0x6e, 0x43, 0x11, 0x2b, 0xdc, 0x05, - 0x15, 0xce, 0x5e, 0xe2, 0xec, 0xb7, 0x0a, 0xd9, 0x45, 0xd0, 0x2a, 0xc2, 0x3f, 0x7b, 0xe3, 0x31, - 0x25, 0x16, 0x73, 0xaf, 0x73, 0x49, 0x50, 0x57, 0xb6, 0x30, 0xc5, 0x88, 0x13, 0xc1, 0x97, 0x41, - 0xcd, 0x15, 0xee, 0xd7, 0xcb, 0xd7, 0x95, 0x8d, 0x72, 0xe7, 0xb2, 0x40, 0xd5, 0xc2, 0xb0, 0x50, - 0x84, 0x68, 0xfd, 0x45, 0x01, 0x6b, 0xf9, 0xb8, 0xb7, 0x75, 0x8f, 0xc2, 0x77, 0x72, 0xb1, 0xab, - 0xd3, 0xc5, 0xce, 0xac, 0x79, 0xe4, 0xd1, 0xc0, 0x61, 0x4b, 0x22, 0xee, 0xb7, 0x41, 0x55, 0xa7, - 0xc4, 0xf4, 0xea, 0xa5, 0xeb, 0xe5, 0x8d, 0x85, 0x3b, 0x37, 0xd4, 0x7c, 0xee, 0xaa, 0x79, 0xc7, - 0x3a, 0x8b, 0x82, 0xb2, 0xfa, 0x16, 0x33, 0x46, 0x01, 0x47, 0xeb, 0xbf, 0x0a, 0x98, 0xdf, 0xc2, - 0xc4, 0xb4, 0xad, 0x7d, 0x42, 0x2f, 0x60, 0xd1, 0xba, 0xa0, 0xe2, 0x39, 0xa4, 0x2f, 0x16, 0xed, - 0x6b, 0x32, 0xdf, 0x23, 0x77, 0xf6, 0x1d, 0xd2, 0x8f, 0x17, 0x8a, 0x3d, 0x21, 0x6e, 0x0c, 0xdf, - 0x06, 0xb3, 0x1e, 0xc5, 0xd4, 0xf7, 0xf8, 0x32, 0x2d, 0xdc, 0xf9, 0xfa, 0xe9, 0x34, 0x1c, 0xda, - 0x59, 0x12, 0x44, 0xb3, 0xc1, 0x33, 0x12, 0x14, 0xad, 0x7f, 0x96, 0x00, 0x8c, 0xb0, 0x5d, 0xdb, - 0xd2, 0x74, 0xca, 0xf2, 0xf7, 0x75, 0x50, 0xa1, 0x23, 0x87, 0xf0, 0x69, 0x98, 0xef, 0xdc, 0x08, - 0xbd, 0xb8, 0x3f, 0x72, 0xc8, 0xc7, 0x27, 0xcd, 0xb5, 0xbc, 0x05, 0xeb, 0x41, 0xdc, 0x06, 0x6e, - 0x47, 0xfe, 0x95, 0xb8, 0xf5, 0xdd, 0xf4, 0xd0, 0x1f, 0x9f, 0x34, 0x25, 0x87, 0x85, 0x1a, 0x31, - 0xa5, 0x1d, 0x84, 0x43, 0x00, 0x0d, 0xec, 0xd1, 0xfb, 0x2e, 0xb6, 0xbc, 0x60, 0x24, 0xdd, 0x24, - 0x22, 0xf2, 0x97, 0xa6, 0x5b, 0x1e, 0x66, 0xd1, 0xb9, 0x2a, 0xbc, 0x80, 0xdb, 0x39, 0x36, 0x24, - 0x19, 0x01, 0xde, 0x00, 0xb3, 0x2e, 0xc1, 0x9e, 0x6d, 0xd5, 0x2b, 0x3c, 0x8a, 0x68, 0x02, 0x11, - 0x6f, 0x45, 0xa2, 0x17, 0xbe, 0x08, 0xe6, 0x4c, 0xe2, 0x79, 0x78, 0x40, 0xea, 0x55, 0x0e, 0x5c, - 0x16, 0xc0, 0xb9, 0x5e, 0xd0, 0x8c, 0xc2, 0xfe, 0xd6, 0xef, 0x15, 0xb0, 0x18, 0xcd, 0xdc, 0x05, - 0x6c, 0x95, 0x4e, 0x7a, 0xab, 0x3c, 0x7f, 0x6a, 0x9e, 0x14, 0xec, 0x90, 0xf7, 0xcb, 0x09, 0x9f, - 0x59, 0x12, 0xc2, 0x9f, 0x80, 0x9a, 0x47, 0x0c, 0xd2, 0xa7, 0xb6, 0x2b, 0x7c, 0x7e, 0x65, 0x4a, - 0x9f, 0xf1, 0x21, 0x31, 0xf6, 0x85, 0x69, 0xe7, 0x12, 0x73, 0x3a, 0x7c, 0x42, 0x11, 0x25, 0xfc, - 0x11, 0xa8, 0x51, 0x62, 0x3a, 0x06, 0xa6, 0x44, 0x6c, 0x93, 0x54, 0x7e, 0xb3, 0x74, 0x61, 0x64, - 0x7b, 0xb6, 0x76, 0x5f, 0xc0, 0xf8, 0x46, 0x89, 0xe6, 0x21, 0x6c, 0x45, 0x11, 0x0d, 0x3c, 0x06, - 0x4b, 0xbe, 0xa3, 0x31, 0x24, 0x65, 0x47, 0xf7, 0x60, 0x24, 0xd2, 0xe7, 0xe6, 0xa9, 0x13, 0x72, - 0x90, 0x32, 0xe9, 0xac, 0x89, 0x01, 0x96, 0xd2, 0xed, 0x28, 0x43, 0x0d, 0x37, 0xc1, 0xb2, 0xa9, - 0x5b, 0x88, 0x60, 0x6d, 0xb4, 0x4f, 0xfa, 0xb6, 0xa5, 0x79, 0x3c, 0x81, 0xaa, 0x9d, 0x75, 0x41, - 0xb0, 0xdc, 0x4b, 0x77, 0xa3, 0x2c, 0x1e, 0x6e, 0x83, 0xd5, 0xf0, 0x9c, 0xfd, 0x81, 0xee, 0x51, - 0xdb, 0x1d, 0x6d, 0xeb, 0xa6, 0x4e, 0xeb, 0xb3, 0x9c, 0xa7, 0x3e, 0x3e, 0x69, 0xae, 0x22, 0x49, - 0x3f, 0x92, 0x5a, 0xb5, 0x7e, 0x33, 0x0b, 0x96, 0x33, 0xa7, 0x01, 0x7c, 0x00, 0xd6, 0xfa, 0xbe, - 0xeb, 0x12, 0x8b, 0xee, 0xf8, 0xe6, 0x21, 0x71, 0xf7, 0xfb, 0x47, 0x44, 0xf3, 0x0d, 0xa2, 0xf1, - 0x15, 0xad, 0x76, 0x1a, 0xc2, 0xd7, 0xb5, 0xae, 0x14, 0x85, 0x0a, 0xac, 0xe1, 0x0f, 0x01, 0xb4, - 0x78, 0x53, 0x4f, 0xf7, 0xbc, 0x88, 0xb3, 0xc4, 0x39, 0xa3, 0x0d, 0xb8, 0x93, 0x43, 0x20, 0x89, - 0x15, 0xf3, 0x51, 0x23, 0x9e, 0xee, 0x12, 0x2d, 0xeb, 0x63, 0x39, 0xed, 0xe3, 0x96, 0x14, 0x85, - 0x0a, 0xac, 0xe1, 0xab, 0x60, 0x21, 0x18, 0x8d, 0xcf, 0xb9, 0x58, 0x9c, 0x15, 0x41, 0xb6, 0xb0, - 0x13, 0x77, 0xa1, 0x24, 0x8e, 0x85, 0x66, 0x1f, 0x7a, 0xc4, 0x1d, 0x12, 0xed, 0xcd, 0x40, 0x03, - 0xb0, 0x42, 0x59, 0xe5, 0x85, 0x32, 0x0a, 0x6d, 0x37, 0x87, 0x40, 0x12, 0x2b, 0x16, 0x5a, 0x90, - 0x35, 0xb9, 0xd0, 0x66, 0xd3, 0xa1, 0x1d, 0x48, 0x51, 0xa8, 0xc0, 0x9a, 0xe5, 0x5e, 0xe0, 0xf2, - 0xe6, 0x10, 0xeb, 0x06, 0x3e, 0x34, 0x48, 0x7d, 0x2e, 0x9d, 0x7b, 0x3b, 0xe9, 0x6e, 0x94, 0xc5, - 0xc3, 0x37, 0xc1, 0x95, 0xa0, 0xe9, 0xc0, 0xc2, 0x11, 0x49, 0x8d, 0x93, 0x3c, 0x27, 0x48, 0xae, - 0xec, 0x64, 0x01, 0x28, 0x6f, 0x03, 0x5f, 0x07, 0x4b, 0x7d, 0xdb, 0x30, 0x78, 0x3e, 0x76, 0x6d, - 0xdf, 0xa2, 0xf5, 0x79, 0xce, 0x02, 0xd9, 0x1e, 0xea, 0xa6, 0x7a, 0x50, 0x06, 0x09, 0x1f, 0x02, - 0xd0, 0x0f, 0xcb, 0x81, 0x57, 0x07, 0xc5, 0x85, 0x3e, 0x5f, 0x87, 0xe2, 0x02, 0x1c, 0x35, 0x79, - 0x28, 0xc1, 0xd6, 0x7a, 0x5f, 0x01, 0xeb, 0x05, 0x7b, 0x1c, 0x7e, 0x2f, 0x55, 0xf5, 0x6e, 0x66, - 0xaa, 0xde, 0xb5, 0x02, 0xb3, 0x44, 0xe9, 0xeb, 0x83, 0x45, 0xa6, 0x3b, 0x74, 0x6b, 0x10, 0x40, - 0xc4, 0x09, 0xf6, 0x92, 0xcc, 0x77, 0x94, 0x04, 0xc6, 0xc7, 0xf0, 0x95, 0xf1, 0x49, 0x73, 0x31, - 0xd5, 0x87, 0xd2, 0x9c, 0xad, 0x5f, 0x96, 0x00, 0xd8, 0x22, 0x8e, 0x61, 0x8f, 0x4c, 0x62, 0x5d, - 0x84, 0x6a, 0xd9, 0x4a, 0xa9, 0x96, 0x96, 0x74, 0x21, 0x22, 0x7f, 0x0a, 0x65, 0xcb, 0x76, 0x46, - 0xb6, 0x7c, 0x63, 0x02, 0xcf, 0xe9, 0xba, 0xe5, 0xef, 0x65, 0xb0, 0x12, 0x83, 0x63, 0xe1, 0x72, - 0x2f, 0xb5, 0x84, 0x2f, 0x64, 0x96, 0x70, 0x5d, 0x62, 0xf2, 0xa9, 0x29, 0x97, 0x77, 0xc1, 0x12, - 0xd3, 0x15, 0xc1, 0xaa, 0x71, 0xd5, 0x32, 0x7b, 0x66, 0xd5, 0x12, 0x55, 0x9d, 0xed, 0x14, 0x13, - 0xca, 0x30, 0x17, 0xa8, 0xa4, 0xb9, 0xcf, 0xa3, 0x4a, 0xfa, 0x83, 0x02, 0x96, 0xe2, 0x65, 0xba, - 0x00, 0x99, 0xd4, 0x4d, 0xcb, 0xa4, 0xc6, 0xe9, 0x79, 0x59, 0xa0, 0x93, 0xfe, 0x56, 0x49, 0x7a, - 0xcd, 0x85, 0xd2, 0x06, 0x7b, 0xa1, 0x72, 0x0c, 0xbd, 0x8f, 0x3d, 0x51, 0x56, 0x2f, 0x05, 0x2f, - 0x53, 0x41, 0x1b, 0x8a, 0x7a, 0x53, 0x92, 0xaa, 0xf4, 0xe9, 0x4a, 0xaa, 0xf2, 0x27, 0x23, 0xa9, - 0xee, 0x83, 0x9a, 0x17, 0x8a, 0xa9, 0x0a, 0xa7, 0xbc, 0x31, 0x69, 0x3b, 0x0b, 0x1d, 0x15, 0xb1, - 0x46, 0x0a, 0x2a, 0x62, 0x92, 0x69, 0xa7, 0xea, 0x67, 0xa9, 0x9d, 0x58, 0x7a, 0x3b, 0xd8, 0xf7, - 0x88, 0xc6, 0xb7, 0x52, 0x2d, 0x4e, 0xef, 0x3d, 0xde, 0x8a, 0x44, 0x2f, 0x3c, 0x00, 0xeb, 0x8e, - 0x6b, 0x0f, 0x5c, 0xe2, 0x79, 0x5b, 0x04, 0x6b, 0x86, 0x6e, 0x91, 0x30, 0x80, 0xa0, 0xea, 0x5d, - 0x1b, 0x9f, 0x34, 0xd7, 0xf7, 0xe4, 0x10, 0x54, 0x64, 0xdb, 0xfa, 0x53, 0x05, 0x5c, 0xce, 0x9e, - 0x88, 0x05, 0x42, 0x44, 0x39, 0x97, 0x10, 0x79, 0x39, 0x91, 0xa2, 0x81, 0x4a, 0x4b, 0xbc, 0xf3, - 0xe7, 0xd2, 0x74, 0x13, 0x2c, 0x0b, 0xe1, 0x11, 0x76, 0x0a, 0x29, 0x16, 0x2d, 0xcf, 0x41, 0xba, - 0x1b, 0x65, 0xf1, 0xf0, 0x1e, 0x58, 0x74, 0xb9, 0xb6, 0x0a, 0x09, 0x02, 0x7d, 0xf2, 0x15, 0x41, - 0xb0, 0x88, 0x92, 0x9d, 0x28, 0x8d, 0x65, 0xda, 0x24, 0x96, 0x1c, 0x21, 0x41, 0x25, 0xad, 0x4d, - 0x36, 0xb3, 0x00, 0x94, 0xb7, 0x81, 0x3d, 0xb0, 0xe2, 0x5b, 0x79, 0xaa, 0x20, 0xd7, 0xae, 0x09, - 0xaa, 0x95, 0x83, 0x3c, 0x04, 0xc9, 0xec, 0xe0, 0x8f, 0x53, 0x72, 0x65, 0x96, 0x9f, 0x22, 0x2f, - 0x9c, 0xbe, 0x1d, 0xa6, 0xd6, 0x2b, 0x12, 0x1d, 0x55, 0x9b, 0x56, 0x47, 0xb5, 0xde, 0x53, 0x00, - 0xcc, 0x6f, 0xc1, 0x89, 0x2f, 0xf7, 0x39, 0x8b, 0x44, 0x89, 0xd4, 0xe4, 0x0a, 0xe7, 0xe6, 0x64, - 0x85, 0x13, 0x9f, 0xa0, 0xd3, 0x49, 0x1c, 0x31, 0xbd, 0x17, 0x73, 0x31, 0x33, 0x85, 0xc4, 0x89, - 0xfd, 0x79, 0x36, 0x89, 0x93, 0xe0, 0x39, 0x5d, 0xe2, 0xfc, 0xab, 0x04, 0x56, 0x62, 0xf0, 0xd4, - 0x12, 0x47, 0x62, 0xf2, 0xe5, 0xe5, 0xcc, 0x74, 0xb2, 0x23, 0x9e, 0xba, 0xff, 0x13, 0xd9, 0x11, - 0x3b, 0x54, 0x20, 0x3b, 0x7e, 0x57, 0x4a, 0x7a, 0x7d, 0x46, 0xd9, 0xf1, 0x09, 0x5c, 0x55, 0x7c, - 0xee, 0x94, 0x4b, 0xeb, 0xcf, 0x65, 0x70, 0x39, 0xbb, 0x05, 0x53, 0x75, 0x50, 0x99, 0x58, 0x07, - 0xf7, 0xc0, 0xea, 0x23, 0xdf, 0x30, 0x46, 0x3c, 0x86, 0x44, 0x31, 0x0c, 0x2a, 0xe8, 0x57, 0x85, - 0xe5, 0xea, 0xf7, 0x25, 0x18, 0x24, 0xb5, 0xcc, 0x97, 0xc5, 0xca, 0xb3, 0x96, 0xc5, 0xea, 0x39, - 0xca, 0xa2, 0x5c, 0x59, 0x94, 0xcf, 0xa5, 0x2c, 0xa6, 0xae, 0x89, 0x92, 0xe3, 0x6a, 0xe2, 0x3b, - 0xfc, 0x58, 0x01, 0x6b, 0xf2, 0xd7, 0x67, 0x68, 0x80, 0x25, 0x13, 0x3f, 0x4e, 0x5e, 0x5e, 0x4c, - 0x2a, 0x18, 0x3e, 0xd5, 0x0d, 0x35, 0xf8, 0xba, 0xa3, 0xbe, 0x65, 0xd1, 0x5d, 0x77, 0x9f, 0xba, - 0xba, 0x35, 0x08, 0x0a, 0x6c, 0x2f, 0xc5, 0x85, 0x32, 0xdc, 0xf0, 0x21, 0xa8, 0x99, 0xf8, 0xf1, - 0xbe, 0xef, 0x0e, 0xc2, 0x42, 0x78, 0xf6, 0x71, 0x78, 0xee, 0xf7, 0x04, 0x0b, 0x8a, 0xf8, 0x5a, - 0x1f, 0x29, 0x60, 0xbd, 0xa0, 0x82, 0x7e, 0x81, 0xa2, 0xdc, 0x05, 0xd7, 0x53, 0x41, 0xb2, 0x0d, - 0x49, 0x1e, 0xf9, 0x06, 0xdf, 0x9b, 0x42, 0xaf, 0xdc, 0x04, 0xf3, 0x0e, 0x76, 0xa9, 0x1e, 0x09, - 0xdd, 0x6a, 0x67, 0x71, 0x7c, 0xd2, 0x9c, 0xdf, 0x0b, 0x1b, 0x51, 0xdc, 0xdf, 0xfa, 0x55, 0x09, - 0x2c, 0x24, 0x48, 0x2e, 0x40, 0x3b, 0xbc, 0x91, 0xd2, 0x0e, 0xd2, 0xaf, 0x31, 0xc9, 0xa8, 0x8a, - 0xc4, 0x43, 0x2f, 0x23, 0x1e, 0xbe, 0x39, 0x89, 0xe8, 0x74, 0xf5, 0xf0, 0xef, 0x12, 0x58, 0x4d, - 0xa0, 0x63, 0xf9, 0xf0, 0x9d, 0x94, 0x7c, 0xd8, 0xc8, 0xc8, 0x87, 0xba, 0xcc, 0xe6, 0x4b, 0xfd, - 0x30, 0x59, 0x3f, 0xfc, 0x51, 0x01, 0xcb, 0x89, 0xb9, 0xbb, 0x00, 0x01, 0xb1, 0x95, 0x16, 0x10, - 0xcd, 0x09, 0xf9, 0x52, 0xa0, 0x20, 0x7e, 0x3d, 0x9b, 0xf2, 0xfb, 0x0b, 0x7f, 0x73, 0xf1, 0x73, - 0xb0, 0x3a, 0xb4, 0x0d, 0xdf, 0x24, 0x5d, 0x03, 0xeb, 0x66, 0x08, 0x60, 0x15, 0x97, 0x4d, 0xe2, - 0x8b, 0x52, 0x7a, 0xe2, 0x7a, 0xba, 0x47, 0x89, 0x45, 0x1f, 0xc4, 0x96, 0x71, 0x9d, 0x7f, 0x20, - 0xa1, 0x43, 0xd2, 0x41, 0xe0, 0xab, 0x60, 0x81, 0x55, 0x4a, 0xbd, 0x4f, 0x76, 0xb0, 0x19, 0xe6, - 0x54, 0xf4, 0xed, 0x61, 0x3f, 0xee, 0x42, 0x49, 0x1c, 0x3c, 0x02, 0x2b, 0x8e, 0xad, 0xf5, 0xb0, - 0x85, 0x07, 0x84, 0x9d, 0xff, 0x7b, 0xb6, 0xa1, 0xf7, 0x47, 0xfc, 0x4e, 0x63, 0xbe, 0xf3, 0x5a, - 0xf8, 0xbe, 0xba, 0x97, 0x87, 0xb0, 0xf7, 0x01, 0x49, 0x33, 0xdf, 0xcf, 0x32, 0x4a, 0x68, 0xe6, - 0x3e, 0x95, 0xcd, 0xe5, 0xfe, 0x5f, 0x20, 0x4b, 0xae, 0x73, 0x7e, 0x2c, 0x2b, 0xba, 0xad, 0xa9, - 0x9d, 0xeb, 0xb6, 0x46, 0xa2, 0x67, 0xe7, 0xcf, 0xa6, 0x67, 0x5b, 0xef, 0x55, 0xc1, 0x95, 0xdc, - 0x19, 0xfb, 0x19, 0x5e, 0xb9, 0xe4, 0x84, 0x61, 0xf9, 0x0c, 0xc2, 0x70, 0x13, 0x2c, 0x8b, 0xef, - 0x74, 0x19, 0x5d, 0x19, 0xcd, 0x47, 0x37, 0xdd, 0x8d, 0xb2, 0x78, 0xd9, 0x95, 0x4f, 0xf5, 0x8c, - 0x57, 0x3e, 0x49, 0x2f, 0xc4, 0xdf, 0x4b, 0x82, 0xc4, 0xcd, 0x7b, 0x21, 0xfe, 0x65, 0x92, 0xc5, - 0xc3, 0xef, 0x86, 0x59, 0x19, 0x31, 0xcc, 0x71, 0x86, 0x4c, 0x9a, 0x45, 0x04, 0x19, 0xf4, 0x33, - 0x7d, 0x8b, 0x7a, 0x47, 0xf2, 0x2d, 0x6a, 0x63, 0xc2, 0x6e, 0x98, 0xfe, 0x76, 0x47, 0xaa, 0xdd, - 0x17, 0xce, 0xae, 0xdd, 0x5b, 0x7f, 0x55, 0xc0, 0x73, 0x85, 0xfb, 0x11, 0x6e, 0xa6, 0x6a, 0xfe, - 0xad, 0x4c, 0xcd, 0x7f, 0xbe, 0xd0, 0x30, 0x51, 0xf8, 0x4d, 0xf9, 0xc5, 0xcf, 0xdd, 0x89, 0x17, - 0x3f, 0x12, 0x45, 0x37, 0xf9, 0x06, 0xa8, 0xb3, 0xf1, 0xe4, 0x69, 0x63, 0xe6, 0x83, 0xa7, 0x8d, - 0x99, 0x0f, 0x9f, 0x36, 0x66, 0x7e, 0x31, 0x6e, 0x28, 0x4f, 0xc6, 0x0d, 0xe5, 0x83, 0x71, 0x43, - 0xf9, 0x70, 0xdc, 0x50, 0xfe, 0x31, 0x6e, 0x28, 0xbf, 0xfd, 0xa8, 0x31, 0xf3, 0xb0, 0x34, 0xbc, - 0xfd, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x69, 0x8a, 0x39, 0xfa, 0x26, 0x00, 0x00, + // 2149 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcb, 0x6f, 0x1b, 0xc7, + 0x19, 0xd7, 0xf2, 0x21, 0x51, 0x23, 0x4b, 0xb2, 0x47, 0xaa, 0xc4, 0xd8, 0x0d, 0xe9, 0xb2, 0xae, + 0xa3, 0xc4, 0x31, 0x59, 0x3b, 0x4e, 0x50, 0xc4, 0x45, 0x02, 0x91, 0x4a, 0xd3, 0x34, 0x7a, 0x75, + 0x64, 0x39, 0x80, 0x9b, 0x16, 0x1d, 0x2d, 0xc7, 0xd4, 0x46, 0xfb, 0xc2, 0xee, 0x2c, 0x63, 0xa2, + 0x97, 0xa2, 0x40, 0x6f, 0x3d, 0xf4, 0x3f, 0x29, 0x8a, 0xa2, 0xb9, 0x15, 0x41, 0xd0, 0x8b, 0x2f, + 0x45, 0x83, 0x5e, 0x9a, 0x13, 0x51, 0x33, 0xa7, 0xa2, 0xe8, 0xad, 0xbd, 0xf8, 0xd2, 0x62, 0x66, + 0x67, 0xdf, 0xb3, 0x22, 0x25, 0x27, 0xca, 0x03, 0xbe, 0x89, 0x33, 0xbf, 0xef, 0x37, 0xdf, 0xcc, + 0x7c, 0xdf, 0x7c, 0xbf, 0x99, 0x15, 0xb8, 0x7d, 0xf4, 0x3d, 0xb7, 0xa9, 0x59, 0xad, 0x23, 0xef, + 0x80, 0x38, 0x26, 0xa1, 0xc4, 0x6d, 0xf5, 0x89, 0xd9, 0xb5, 0x9c, 0x96, 0xe8, 0xc0, 0xb6, 0xd6, + 0xc2, 0xb6, 0xed, 0xb6, 0xfa, 0x37, 0x5a, 0x3d, 0x62, 0x12, 0x07, 0x53, 0xd2, 0x6d, 0xda, 0x8e, + 0x45, 0x2d, 0x08, 0x7d, 0x4c, 0x13, 0xdb, 0x5a, 0x93, 0x61, 0x9a, 0xfd, 0x1b, 0x17, 0xaf, 0xf7, + 0x34, 0x7a, 0xe8, 0x1d, 0x34, 0x55, 0xcb, 0x68, 0xf5, 0xac, 0x9e, 0xd5, 0xe2, 0xd0, 0x03, 0xef, + 0x3e, 0xff, 0xc5, 0x7f, 0xf0, 0xbf, 0x7c, 0x8a, 0x8b, 0x8d, 0xd8, 0x30, 0xaa, 0xe5, 0x10, 0xc9, + 0x30, 0x17, 0x6f, 0x45, 0x18, 0x03, 0xab, 0x87, 0x9a, 0x49, 0x9c, 0x41, 0xcb, 0x3e, 0xea, 0xb1, + 0x06, 0xb7, 0x65, 0x10, 0x8a, 0x65, 0x56, 0xad, 0x3c, 0x2b, 0xc7, 0x33, 0xa9, 0x66, 0x90, 0x8c, + 0xc1, 0x2b, 0xe3, 0x0c, 0x5c, 0xf5, 0x90, 0x18, 0x38, 0x63, 0xf7, 0x52, 0x9e, 0x9d, 0x47, 0x35, + 0xbd, 0xa5, 0x99, 0xd4, 0xa5, 0x4e, 0xda, 0xa8, 0xf1, 0x5f, 0x05, 0xc0, 0x8e, 0x65, 0x52, 0xc7, + 0xd2, 0x75, 0xe2, 0x20, 0xd2, 0xd7, 0x5c, 0xcd, 0x32, 0xe1, 0xcf, 0x41, 0x85, 0xcd, 0xa7, 0x8b, + 0x29, 0xae, 0x2a, 0x97, 0x95, 0xb5, 0xb9, 0x9b, 0xdf, 0x6d, 0x46, 0x8b, 0x1c, 0xd2, 0x37, 0xed, + 0xa3, 0x1e, 0x6b, 0x70, 0x9b, 0x0c, 0xdd, 0xec, 0xdf, 0x68, 0xee, 0x1c, 0xbc, 0x47, 0x54, 0xba, + 0x45, 0x28, 0x6e, 0xc3, 0x87, 0xc3, 0xfa, 0xd4, 0x68, 0x58, 0x07, 0x51, 0x1b, 0x0a, 0x59, 0xe1, + 0x0e, 0x28, 0x71, 0xf6, 0x02, 0x67, 0xbf, 0x9e, 0xcb, 0x2e, 0x26, 0xdd, 0x44, 0xf8, 0xfd, 0x37, + 0x1e, 0x50, 0x62, 0x32, 0xf7, 0xda, 0xe7, 0x04, 0x75, 0x69, 0x03, 0x53, 0x8c, 0x38, 0x11, 0x7c, + 0x11, 0x54, 0x1c, 0xe1, 0x7e, 0xb5, 0x78, 0x59, 0x59, 0x2b, 0xb6, 0xcf, 0x0b, 0x54, 0x25, 0x98, + 0x16, 0x0a, 0x11, 0x8d, 0x3f, 0x2b, 0x60, 0x25, 0x3b, 0xef, 0x4d, 0xcd, 0xa5, 0xf0, 0xdd, 0xcc, + 0xdc, 0x9b, 0x93, 0xcd, 0x9d, 0x59, 0xf3, 0x99, 0x87, 0x03, 0x07, 0x2d, 0xb1, 0x79, 0xbf, 0x0d, + 0xca, 0x1a, 0x25, 0x86, 0x5b, 0x2d, 0x5c, 0x2e, 0xae, 0xcd, 0xdd, 0xbc, 0xda, 0xcc, 0xc6, 0x6e, + 0x33, 0xeb, 0x58, 0x7b, 0x5e, 0x50, 0x96, 0xdf, 0x62, 0xc6, 0xc8, 0xe7, 0x68, 0xfc, 0x4f, 0x01, + 0xb3, 0x1b, 0x98, 0x18, 0x96, 0xb9, 0x47, 0xe8, 0x19, 0x6c, 0x5a, 0x07, 0x94, 0x5c, 0x9b, 0xa8, + 0x62, 0xd3, 0xbe, 0x25, 0xf3, 0x3d, 0x74, 0x67, 0xcf, 0x26, 0x6a, 0xb4, 0x51, 0xec, 0x17, 0xe2, + 0xc6, 0xf0, 0x6d, 0x30, 0xed, 0x52, 0x4c, 0x3d, 0x97, 0x6f, 0xd3, 0xdc, 0xcd, 0x6f, 0x1f, 0x4f, + 0xc3, 0xa1, 0xed, 0x05, 0x41, 0x34, 0xed, 0xff, 0x46, 0x82, 0xa2, 0xf1, 0xcf, 0x02, 0x80, 0x21, + 0xb6, 0x63, 0x99, 0x5d, 0x8d, 0xb2, 0xf8, 0x7d, 0x15, 0x94, 0xe8, 0xc0, 0x26, 0x7c, 0x19, 0x66, + 0xdb, 0x57, 0x03, 0x2f, 0xee, 0x0c, 0x6c, 0xf2, 0x78, 0x58, 0x5f, 0xc9, 0x5a, 0xb0, 0x1e, 0xc4, + 0x6d, 0xe0, 0x66, 0xe8, 0x5f, 0x81, 0x5b, 0xdf, 0x4a, 0x0e, 0xfd, 0x78, 0x58, 0x97, 0x1c, 0x16, + 0xcd, 0x90, 0x29, 0xe9, 0x20, 0xec, 0x03, 0xa8, 0x63, 0x97, 0xde, 0x71, 0xb0, 0xe9, 0xfa, 0x23, + 0x69, 0x06, 0x11, 0x33, 0x7f, 0x61, 0xb2, 0xed, 0x61, 0x16, 0xed, 0x8b, 0xc2, 0x0b, 0xb8, 0x99, + 0x61, 0x43, 0x92, 0x11, 0xe0, 0x55, 0x30, 0xed, 0x10, 0xec, 0x5a, 0x66, 0xb5, 0xc4, 0x67, 0x11, + 0x2e, 0x20, 0xe2, 0xad, 0x48, 0xf4, 0xc2, 0xe7, 0xc1, 0x8c, 0x41, 0x5c, 0x17, 0xf7, 0x48, 0xb5, + 0xcc, 0x81, 0x8b, 0x02, 0x38, 0xb3, 0xe5, 0x37, 0xa3, 0xa0, 0xbf, 0xf1, 0x7b, 0x05, 0xcc, 0x87, + 0x2b, 0x77, 0x06, 0xa9, 0xd2, 0x4e, 0xa6, 0xca, 0xb3, 0xc7, 0xc6, 0x49, 0x4e, 0x86, 0x7c, 0x58, + 0x8c, 0xf9, 0xcc, 0x82, 0x10, 0xfe, 0x14, 0x54, 0x5c, 0xa2, 0x13, 0x95, 0x5a, 0x8e, 0xf0, 0xf9, + 0xa5, 0x09, 0x7d, 0xc6, 0x07, 0x44, 0xdf, 0x13, 0xa6, 0xed, 0x73, 0xcc, 0xe9, 0xe0, 0x17, 0x0a, + 0x29, 0xe1, 0x8f, 0x41, 0x85, 0x12, 0xc3, 0xd6, 0x31, 0x25, 0x22, 0x4d, 0x12, 0xf1, 0xcd, 0xc2, + 0x85, 0x91, 0xed, 0x5a, 0xdd, 0x3b, 0x02, 0xc6, 0x13, 0x25, 0x5c, 0x87, 0xa0, 0x15, 0x85, 0x34, + 0xf0, 0x08, 0x2c, 0x78, 0x76, 0x97, 0x21, 0x29, 0x3b, 0xba, 0x7b, 0x03, 0x11, 0x3e, 0xd7, 0x8e, + 0x5d, 0x90, 0xfd, 0x84, 0x49, 0x7b, 0x45, 0x0c, 0xb0, 0x90, 0x6c, 0x47, 0x29, 0x6a, 0xb8, 0x0e, + 0x16, 0x0d, 0xcd, 0x44, 0x04, 0x77, 0x07, 0x7b, 0x44, 0xb5, 0xcc, 0xae, 0xcb, 0x03, 0xa8, 0xdc, + 0x5e, 0x15, 0x04, 0x8b, 0x5b, 0xc9, 0x6e, 0x94, 0xc6, 0xc3, 0x4d, 0xb0, 0x1c, 0x9c, 0xb3, 0x3f, + 0xd4, 0x5c, 0x6a, 0x39, 0x83, 0x4d, 0xcd, 0xd0, 0x68, 0x75, 0x9a, 0xf3, 0x54, 0x47, 0xc3, 0xfa, + 0x32, 0x92, 0xf4, 0x23, 0xa9, 0x55, 0xe3, 0x37, 0xd3, 0x60, 0x31, 0x75, 0x1a, 0xc0, 0xbb, 0x60, + 0x45, 0xf5, 0x1c, 0x87, 0x98, 0x74, 0xdb, 0x33, 0x0e, 0x88, 0xb3, 0xa7, 0x1e, 0x92, 0xae, 0xa7, + 0x93, 0x2e, 0xdf, 0xd1, 0x72, 0xbb, 0x26, 0x7c, 0x5d, 0xe9, 0x48, 0x51, 0x28, 0xc7, 0x1a, 0xfe, + 0x08, 0x40, 0x93, 0x37, 0x6d, 0x69, 0xae, 0x1b, 0x72, 0x16, 0x38, 0x67, 0x98, 0x80, 0xdb, 0x19, + 0x04, 0x92, 0x58, 0x31, 0x1f, 0xbb, 0xc4, 0xd5, 0x1c, 0xd2, 0x4d, 0xfb, 0x58, 0x4c, 0xfa, 0xb8, + 0x21, 0x45, 0xa1, 0x1c, 0x6b, 0xf8, 0x32, 0x98, 0xf3, 0x47, 0xe3, 0x6b, 0x2e, 0x36, 0x67, 0x49, + 0x90, 0xcd, 0x6d, 0x47, 0x5d, 0x28, 0x8e, 0x63, 0x53, 0xb3, 0x0e, 0x5c, 0xe2, 0xf4, 0x49, 0xf7, + 0x4d, 0x5f, 0x03, 0xb0, 0x42, 0x59, 0xe6, 0x85, 0x32, 0x9c, 0xda, 0x4e, 0x06, 0x81, 0x24, 0x56, + 0x6c, 0x6a, 0x7e, 0xd4, 0x64, 0xa6, 0x36, 0x9d, 0x9c, 0xda, 0xbe, 0x14, 0x85, 0x72, 0xac, 0x59, + 0xec, 0xf9, 0x2e, 0xaf, 0xf7, 0xb1, 0xa6, 0xe3, 0x03, 0x9d, 0x54, 0x67, 0x92, 0xb1, 0xb7, 0x9d, + 0xec, 0x46, 0x69, 0x3c, 0x7c, 0x13, 0x5c, 0xf0, 0x9b, 0xf6, 0x4d, 0x1c, 0x92, 0x54, 0x38, 0xc9, + 0x33, 0x82, 0xe4, 0xc2, 0x76, 0x1a, 0x80, 0xb2, 0x36, 0xf0, 0x55, 0xb0, 0xa0, 0x5a, 0xba, 0xce, + 0xe3, 0xb1, 0x63, 0x79, 0x26, 0xad, 0xce, 0x72, 0x16, 0xc8, 0x72, 0xa8, 0x93, 0xe8, 0x41, 0x29, + 0x24, 0xbc, 0x07, 0x80, 0x1a, 0x94, 0x03, 0xb7, 0x0a, 0xf2, 0x0b, 0x7d, 0xb6, 0x0e, 0x45, 0x05, + 0x38, 0x6c, 0x72, 0x51, 0x8c, 0xad, 0xf1, 0xa1, 0x02, 0x56, 0x73, 0x72, 0x1c, 0xbe, 0x9e, 0xa8, + 0x7a, 0xd7, 0x52, 0x55, 0xef, 0x52, 0x8e, 0x59, 0xac, 0xf4, 0xa9, 0x60, 0x9e, 0xe9, 0x0e, 0xcd, + 0xec, 0xf9, 0x10, 0x71, 0x82, 0xbd, 0x20, 0xf3, 0x1d, 0xc5, 0x81, 0xd1, 0x31, 0x7c, 0x61, 0x34, + 0xac, 0xcf, 0x27, 0xfa, 0x50, 0x92, 0xb3, 0xf1, 0xab, 0x02, 0x00, 0x1b, 0xc4, 0xd6, 0xad, 0x81, + 0x41, 0xcc, 0xb3, 0x50, 0x2d, 0x1b, 0x09, 0xd5, 0xd2, 0x90, 0x6e, 0x44, 0xe8, 0x4f, 0xae, 0x6c, + 0xd9, 0x4c, 0xc9, 0x96, 0x2b, 0x63, 0x78, 0x8e, 0xd7, 0x2d, 0x7f, 0x2f, 0x82, 0xa5, 0x08, 0x1c, + 0x09, 0x97, 0xdb, 0x89, 0x2d, 0x7c, 0x2e, 0xb5, 0x85, 0xab, 0x12, 0x93, 0xcf, 0x4d, 0xb9, 0xbc, + 0x07, 0x16, 0x98, 0xae, 0xf0, 0x77, 0x8d, 0xab, 0x96, 0xe9, 0x13, 0xab, 0x96, 0xb0, 0xea, 0x6c, + 0x26, 0x98, 0x50, 0x8a, 0x39, 0x47, 0x25, 0xcd, 0x7c, 0x15, 0x55, 0xd2, 0x1f, 0x14, 0xb0, 0x10, + 0x6d, 0xd3, 0x19, 0xc8, 0xa4, 0x4e, 0x52, 0x26, 0xd5, 0x8e, 0x8f, 0xcb, 0x1c, 0x9d, 0xf4, 0xb7, + 0x52, 0xdc, 0x6b, 0x2e, 0x94, 0xd6, 0xd8, 0x85, 0xca, 0xd6, 0x35, 0x15, 0xbb, 0xa2, 0xac, 0x9e, + 0xf3, 0x2f, 0x53, 0x7e, 0x1b, 0x0a, 0x7b, 0x13, 0x92, 0xaa, 0xf0, 0xf9, 0x4a, 0xaa, 0xe2, 0x67, + 0x23, 0xa9, 0xee, 0x80, 0x8a, 0x1b, 0x88, 0xa9, 0x12, 0xa7, 0xbc, 0x3a, 0x2e, 0x9d, 0x85, 0x8e, + 0x0a, 0x59, 0x43, 0x05, 0x15, 0x32, 0xc9, 0xb4, 0x53, 0xf9, 0x8b, 0xd4, 0x4e, 0x2c, 0xbc, 0x6d, + 0xec, 0xb9, 0xa4, 0xcb, 0x53, 0xa9, 0x12, 0x85, 0xf7, 0x2e, 0x6f, 0x45, 0xa2, 0x17, 0xee, 0x83, + 0x55, 0xdb, 0xb1, 0x7a, 0x0e, 0x71, 0xdd, 0x0d, 0x82, 0xbb, 0xba, 0x66, 0x92, 0x60, 0x02, 0x7e, + 0xd5, 0xbb, 0x34, 0x1a, 0xd6, 0x57, 0x77, 0xe5, 0x10, 0x94, 0x67, 0xdb, 0xf8, 0x53, 0x09, 0x9c, + 0x4f, 0x9f, 0x88, 0x39, 0x42, 0x44, 0x39, 0x95, 0x10, 0x79, 0x31, 0x16, 0xa2, 0xbe, 0x4a, 0x8b, + 0xdd, 0xf9, 0x33, 0x61, 0xba, 0x0e, 0x16, 0x85, 0xf0, 0x08, 0x3a, 0x85, 0x14, 0x0b, 0xb7, 0x67, + 0x3f, 0xd9, 0x8d, 0xd2, 0x78, 0x78, 0x1b, 0xcc, 0x3b, 0x5c, 0x5b, 0x05, 0x04, 0xbe, 0x3e, 0xf9, + 0x86, 0x20, 0x98, 0x47, 0xf1, 0x4e, 0x94, 0xc4, 0x32, 0x6d, 0x12, 0x49, 0x8e, 0x80, 0xa0, 0x94, + 0xd4, 0x26, 0xeb, 0x69, 0x00, 0xca, 0xda, 0xc0, 0x2d, 0xb0, 0xe4, 0x99, 0x59, 0x2a, 0x3f, 0xd6, + 0x2e, 0x09, 0xaa, 0xa5, 0xfd, 0x2c, 0x04, 0xc9, 0xec, 0xe0, 0x4f, 0x12, 0x72, 0x65, 0x9a, 0x9f, + 0x22, 0xcf, 0x1d, 0x9f, 0x0e, 0x13, 0xeb, 0x15, 0x89, 0x8e, 0xaa, 0x4c, 0xaa, 0xa3, 0x1a, 0x1f, + 0x28, 0x00, 0x66, 0x53, 0x70, 0xec, 0xe5, 0x3e, 0x63, 0x11, 0x2b, 0x91, 0x5d, 0xb9, 0xc2, 0xb9, + 0x36, 0x5e, 0xe1, 0x44, 0x27, 0xe8, 0x64, 0x12, 0x47, 0x2c, 0xef, 0xd9, 0x3c, 0xcc, 0x4c, 0x20, + 0x71, 0x22, 0x7f, 0x9e, 0x4c, 0xe2, 0xc4, 0x78, 0x8e, 0x97, 0x38, 0xff, 0x2a, 0x80, 0xa5, 0x08, + 0x3c, 0xb1, 0xc4, 0x91, 0x98, 0x3c, 0x7d, 0x9c, 0x99, 0x4c, 0x76, 0x44, 0x4b, 0xf7, 0x25, 0x91, + 0x1d, 0x91, 0x43, 0x39, 0xb2, 0xe3, 0x77, 0x85, 0xb8, 0xd7, 0x27, 0x94, 0x1d, 0x9f, 0xc1, 0x53, + 0xc5, 0x57, 0x4e, 0xb9, 0x34, 0x3e, 0x2a, 0x82, 0xf3, 0xe9, 0x14, 0x4c, 0xd4, 0x41, 0x65, 0x6c, + 0x1d, 0xdc, 0x05, 0xcb, 0xf7, 0x3d, 0x5d, 0x1f, 0xf0, 0x39, 0xc4, 0x8a, 0xa1, 0x5f, 0x41, 0xbf, + 0x29, 0x2c, 0x97, 0x7f, 0x20, 0xc1, 0x20, 0xa9, 0x65, 0xb6, 0x2c, 0x96, 0x9e, 0xb4, 0x2c, 0x96, + 0x4f, 0x51, 0x16, 0xe5, 0xca, 0xa2, 0x78, 0x2a, 0x65, 0x31, 0x71, 0x4d, 0x94, 0x1c, 0x57, 0x63, + 0xef, 0xf0, 0x23, 0x05, 0xac, 0xc8, 0xaf, 0xcf, 0x50, 0x07, 0x0b, 0x06, 0x7e, 0x10, 0x7f, 0xbc, + 0x18, 0x57, 0x30, 0x3c, 0xaa, 0xe9, 0x4d, 0xff, 0xeb, 0x4e, 0xf3, 0x2d, 0x93, 0xee, 0x38, 0x7b, + 0xd4, 0xd1, 0xcc, 0x9e, 0x5f, 0x60, 0xb7, 0x12, 0x5c, 0x28, 0xc5, 0x0d, 0xef, 0x81, 0x8a, 0x81, + 0x1f, 0xec, 0x79, 0x4e, 0x2f, 0x28, 0x84, 0x27, 0x1f, 0x87, 0xc7, 0xfe, 0x96, 0x60, 0x41, 0x21, + 0x5f, 0xe3, 0x53, 0x05, 0xac, 0xe6, 0x54, 0xd0, 0xaf, 0xd1, 0x2c, 0x77, 0xc0, 0xe5, 0xc4, 0x24, + 0x59, 0x42, 0x92, 0xfb, 0x9e, 0xce, 0x73, 0x53, 0xe8, 0x95, 0x6b, 0x60, 0xd6, 0xc6, 0x0e, 0xd5, + 0x42, 0xa1, 0x5b, 0x6e, 0xcf, 0x8f, 0x86, 0xf5, 0xd9, 0xdd, 0xa0, 0x11, 0x45, 0xfd, 0x8d, 0x5f, + 0x17, 0xc0, 0x5c, 0x8c, 0xe4, 0x0c, 0xb4, 0xc3, 0x1b, 0x09, 0xed, 0x20, 0xfd, 0x1a, 0x13, 0x9f, + 0x55, 0x9e, 0x78, 0xd8, 0x4a, 0x89, 0x87, 0xef, 0x8c, 0x23, 0x3a, 0x5e, 0x3d, 0xfc, 0xbb, 0x00, + 0x96, 0x63, 0xe8, 0x48, 0x3e, 0x7c, 0x3f, 0x21, 0x1f, 0xd6, 0x52, 0xf2, 0xa1, 0x2a, 0xb3, 0x79, + 0xaa, 0x1f, 0xc6, 0xeb, 0x87, 0x3f, 0x2a, 0x60, 0x31, 0xb6, 0x76, 0x67, 0x20, 0x20, 0x36, 0x92, + 0x02, 0xa2, 0x3e, 0x26, 0x5e, 0x72, 0x14, 0xc4, 0x7f, 0x14, 0xd0, 0x8a, 0xa1, 0x76, 0x89, 0xe3, + 0x6a, 0x2e, 0x25, 0x26, 0xbd, 0x6b, 0xe9, 0x9e, 0x41, 0x3a, 0x3a, 0xd6, 0x0c, 0x44, 0x58, 0x83, + 0x66, 0x99, 0xbb, 0x96, 0xae, 0xa9, 0x03, 0x88, 0xc1, 0xdc, 0xfb, 0x87, 0xc4, 0xdc, 0x20, 0x3a, + 0xa1, 0xe2, 0x9b, 0xc1, 0x6c, 0xfb, 0xf5, 0xe0, 0x09, 0xfd, 0x9d, 0xa8, 0xeb, 0xf1, 0xb0, 0xbe, + 0x36, 0x09, 0x23, 0x0f, 0xb0, 0x38, 0x27, 0xfc, 0x19, 0x00, 0xec, 0xe7, 0x9e, 0x8a, 0x83, 0x2f, + 0x08, 0xb3, 0xed, 0xd7, 0x82, 0x34, 0x7c, 0x27, 0xec, 0x39, 0xd1, 0x00, 0x31, 0xc6, 0xc6, 0x5f, + 0x67, 0x12, 0xdb, 0xf5, 0xb5, 0x7f, 0xb0, 0xf9, 0x05, 0x58, 0xee, 0x47, 0xab, 0x13, 0x00, 0x98, + 0xd0, 0x60, 0xb1, 0xf3, 0xbc, 0x94, 0x5e, 0xb6, 0xae, 0x91, 0xbc, 0xb9, 0x2b, 0xa1, 0x43, 0xd2, + 0x41, 0xe0, 0xcb, 0x60, 0x8e, 0x09, 0x04, 0x4d, 0x25, 0xdb, 0xd8, 0x08, 0x52, 0x29, 0xfc, 0xe4, + 0xb2, 0x17, 0x75, 0xa1, 0x38, 0x0e, 0x1e, 0x82, 0x25, 0xdb, 0xea, 0x6e, 0x61, 0x13, 0xf7, 0x08, + 0x2b, 0x7b, 0xfe, 0x56, 0xf2, 0xa7, 0x9c, 0xd9, 0xf6, 0x2b, 0xc1, 0x35, 0x7d, 0x37, 0x0b, 0x61, + 0xd7, 0x20, 0x49, 0x33, 0x0f, 0x02, 0x19, 0x25, 0x34, 0x32, 0x5f, 0x08, 0x67, 0x32, 0xff, 0x56, + 0x21, 0xcb, 0xa9, 0x53, 0x7e, 0x23, 0xcc, 0x7b, 0xa4, 0xaa, 0x9c, 0xea, 0x91, 0x4a, 0x22, 0xe3, + 0x67, 0x4f, 0x28, 0xe3, 0x3f, 0x52, 0xc0, 0x15, 0x7b, 0x82, 0x34, 0xaa, 0x02, 0xbe, 0x2c, 0x9d, + 0x31, 0xcb, 0x32, 0x49, 0x46, 0xb6, 0xd7, 0x46, 0xc3, 0xfa, 0x95, 0x49, 0x90, 0x68, 0x22, 0xd7, + 0x1a, 0x1f, 0x94, 0xc1, 0x85, 0x4c, 0x79, 0xfc, 0x02, 0x5f, 0xcb, 0x32, 0x9a, 0xbe, 0x78, 0x02, + 0x4d, 0xbf, 0x0e, 0x16, 0xc5, 0x27, 0xd6, 0xd4, 0x95, 0x20, 0xdc, 0xd3, 0x4e, 0xb2, 0x1b, 0xa5, + 0xf1, 0xb2, 0xd7, 0xba, 0xf2, 0x09, 0x5f, 0xeb, 0xe2, 0x5e, 0x88, 0xff, 0x0c, 0xf2, 0x93, 0x2f, + 0xeb, 0x85, 0xf8, 0x07, 0xa1, 0x34, 0x1e, 0xbe, 0x16, 0x64, 0x56, 0xc8, 0x30, 0xc3, 0x19, 0x52, + 0xa9, 0x12, 0x12, 0xa4, 0xd0, 0x4f, 0xf4, 0x19, 0xf1, 0x5d, 0xc9, 0x67, 0xc4, 0xb5, 0x31, 0xa1, + 0x3b, 0xf9, 0xc3, 0x9c, 0xf4, 0xda, 0x35, 0x77, 0xf2, 0x6b, 0x57, 0xe3, 0x2f, 0x0a, 0x78, 0x26, + 0xf7, 0x4c, 0x81, 0xeb, 0x09, 0xb9, 0x76, 0x3d, 0x25, 0xd7, 0x9e, 0xcd, 0x35, 0x8c, 0x69, 0x36, + 0x43, 0xfe, 0x66, 0x77, 0x6b, 0xec, 0x9b, 0x9d, 0x44, 0x8c, 0x8f, 0x7f, 0xbc, 0x6b, 0xaf, 0x3d, + 0x7c, 0x54, 0x9b, 0xfa, 0xf8, 0x51, 0x6d, 0xea, 0x93, 0x47, 0xb5, 0xa9, 0x5f, 0x8e, 0x6a, 0xca, + 0xc3, 0x51, 0x4d, 0xf9, 0x78, 0x54, 0x53, 0x3e, 0x19, 0xd5, 0x94, 0x7f, 0x8c, 0x6a, 0xca, 0x6f, + 0x3f, 0xad, 0x4d, 0xdd, 0x2b, 0xf4, 0x6f, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x7f, 0x6b, + 0xc0, 0xb5, 0x28, 0x00, 0x00, } func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { @@ -2291,6 +2329,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i -= len(m.WhenScaled) + copy(dAtA[i:], m.WhenScaled) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled))) + i-- + dAtA[i] = 0x12 + i -= len(m.WhenDeleted) + copy(dAtA[i:], m.WhenDeleted) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2311,6 +2382,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.PersistentVolumeClaimRetentionPolicy != nil { + { + size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i-- dAtA[i] = 0x48 @@ -2955,6 +3038,19 @@ func (m *StatefulSetList) Size() (n int) { return n } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.WhenDeleted) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.WhenScaled) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *StatefulSetSpec) Size() (n int) { if m == nil { return 0 @@ -2986,6 +3082,10 @@ func (m *StatefulSetSpec) Size() (n int) { n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) } n += 1 + sovGenerated(uint64(m.MinReadySeconds)) + if m.PersistentVolumeClaimRetentionPolicy != nil { + l = m.PersistentVolumeClaimRetentionPolicy.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -3399,6 +3499,17 @@ func (this *StatefulSetList) String() string { }, "") return s } +func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`, + `WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`, + `WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`, + `}`, + }, "") + return s +} func (this *StatefulSetSpec) String() string { if this == nil { return "nil" @@ -3418,6 +3529,7 @@ func (this *StatefulSetSpec) String() string { `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, + `PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`, `}`, }, "") return s @@ -7461,6 +7573,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error { } return nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7749,6 +7975,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PersistentVolumeClaimRetentionPolicy == nil { + m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/apps/v1/generated.proto b/staging/src/k8s.io/api/apps/v1/generated.proto index 8d1e510a5b30..6e5517d81fa8 100644 --- a/staging/src/k8s.io/api/apps/v1/generated.proto +++ b/staging/src/k8s.io/api/apps/v1/generated.proto @@ -611,6 +611,23 @@ message StatefulSetList { repeated StatefulSet items = 2; } +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +message StatefulSetPersistentVolumeClaimRetentionPolicy { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + optional string whenDeleted = 1; + + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + optional string whenScaled = 2; +} + // A StatefulSetSpec is the specification of a StatefulSet. message StatefulSetSpec { // replicas is the desired number of replicas of the given Template. @@ -677,6 +694,15 @@ message StatefulSetSpec { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional optional int32 minReadySeconds = 9; + + // persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent + // volume claims created from volumeClaimTemplates. By default, all persistent + // volume claims are created as needed and retained until manually deleted. This + // policy allows the lifecycle to be altered, for example by deleting persistent + // volume claims when their stateful set is deleted, or when their pod is scaled + // down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, + // which is alpha. +optional + optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10; } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1/types.go b/staging/src/k8s.io/api/apps/v1/types.go index 4a8ebb73c4ba..469b47297f64 100644 --- a/staging/src/k8s.io/api/apps/v1/types.go +++ b/staging/src/k8s.io/api/apps/v1/types.go @@ -118,6 +118,40 @@ type RollingUpdateStatefulSetStrategy struct { Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` } +// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine +// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is +// deleted or scaled down. +type PersistentVolumeClaimRetentionPolicyType string + +const ( + // RetainPersistentVolumeClaimRetentionPolicyType is the default + // PersistentVolumeClaimRetentionPolicy and specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will not be deleted. + RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain" + // RetentionPersistentVolumeClaimRetentionPolicyType specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will be deleted in the scenario specified in + // StatefulSetPersistentVolumeClaimRetentionPolicy. + DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +type StatefulSetPersistentVolumeClaimRetentionPolicy struct { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"` + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"` +} + // A StatefulSetSpec is the specification of a StatefulSet. type StatefulSetSpec struct { // replicas is the desired number of replicas of the given Template. @@ -184,6 +218,15 @@ type StatefulSetSpec struct { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` + + // persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent + // volume claims created from volumeClaimTemplates. By default, all persistent + // volume claims are created as needed and retained until manually deleted. This + // policy allows the lifecycle to be altered, for example by deleting persistent + // volume claims when their stateful set is deleted, or when their pod is scaled + // down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, + // which is alpha. +optional + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"` } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/apps/v1/types_swagger_doc_generated.go index 6f28ef538da1..f640f9cdd661 100644 --- a/staging/src/k8s.io/api/apps/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/apps/v1/types_swagger_doc_generated.go @@ -323,17 +323,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string { return map_StatefulSetList } +var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{ + "": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.", + "whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.", + "whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.", +} + +func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string { + return map_StatefulSetPersistentVolumeClaimRetentionPolicy +} + var map_StatefulSetSpec = map[string]string{ - "": "A StatefulSetSpec is the specification of a StatefulSet.", - "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", - "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", - "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", - "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", - "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", - "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", - "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", - "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", - "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "": "A StatefulSetSpec is the specification of a StatefulSet.", + "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", + "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", + "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", + "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", + "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", + "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", + "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", + "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", + "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "persistentVolumeClaimRetentionPolicy": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional", } func (StatefulSetSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/apps/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/apps/v1/zz_generated.deepcopy.go index ff8fba0463be..8e4d4261a290 100644 --- a/staging/src/k8s.io/api/apps/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/apps/v1/zz_generated.deepcopy.go @@ -688,6 +688,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy { + if in == nil { + return nil + } + out := new(StatefulSetPersistentVolumeClaimRetentionPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in @@ -715,6 +731,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = new(int32) **out = **in } + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = new(StatefulSetPersistentVolumeClaimRetentionPolicy) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go b/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go index 79e39e582f10..74584223c992 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go @@ -553,10 +553,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() { var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() { + *m = StatefulSetPersistentVolumeClaimRetentionPolicy{} +} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) { + return fileDescriptor_2a07313e8f66e805, []int{18} +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int { + return m.Size() +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m) +} + +var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo + func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_2a07313e8f66e805, []int{18} + return fileDescriptor_2a07313e8f66e805, []int{19} } func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -584,7 +614,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_2a07313e8f66e805, []int{19} + return fileDescriptor_2a07313e8f66e805, []int{20} } func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -612,7 +642,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_2a07313e8f66e805, []int{20} + return fileDescriptor_2a07313e8f66e805, []int{21} } func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -658,6 +688,7 @@ func init() { proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta1.StatefulSet") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta1.StatefulSetCondition") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta1.StatefulSetList") + proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta1.StatefulSetSpec") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta1.StatefulSetStatus") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta1.StatefulSetUpdateStrategy") @@ -668,124 +699,130 @@ func init() { } var fileDescriptor_2a07313e8f66e805 = []byte{ - // 1869 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x24, 0x47, - 0x15, 0x77, 0x8f, 0x3d, 0xf6, 0xcc, 0x73, 0x3c, 0xde, 0x2d, 0x9b, 0xf5, 0xc4, 0x81, 0xb1, 0x35, - 0x44, 0x89, 0xf3, 0xe1, 0x9e, 0xac, 0x13, 0xa2, 0x64, 0x17, 0x45, 0x78, 0xbc, 0x4b, 0xb2, 0x91, - 0x8d, 0x9d, 0xb2, 0x1d, 0x44, 0x00, 0x29, 0x35, 0x3d, 0xb5, 0xb3, 0x1d, 0xf7, 0x97, 0xba, 0xab, - 0x87, 0x1d, 0x71, 0xe1, 0x0f, 0x40, 0x4a, 0xce, 0xfc, 0x15, 0xdc, 0x40, 0x70, 0xe3, 0xb4, 0xc7, - 0x88, 0x0b, 0x39, 0x59, 0xec, 0xe4, 0x0a, 0x47, 0x2e, 0x2b, 0x21, 0xa1, 0xaa, 0xae, 0xfe, 0xee, - 0xb6, 0xdb, 0x48, 0x6b, 0x09, 0x6e, 0xd3, 0xf5, 0xde, 0xfb, 0xbd, 0x57, 0x55, 0xef, 0xbd, 0x7a, - 0xbf, 0x81, 0x1f, 0x9d, 0xbd, 0xe7, 0xa9, 0xba, 0xdd, 0x3b, 0xf3, 0x07, 0xd4, 0xb5, 0x28, 0xa3, - 0x5e, 0x6f, 0x4c, 0xad, 0xa1, 0xed, 0xf6, 0xa4, 0x80, 0x38, 0x7a, 0x8f, 0x38, 0x8e, 0xd7, 0x1b, - 0xdf, 0x1e, 0x50, 0x46, 0x6e, 0xf7, 0x46, 0xd4, 0xa2, 0x2e, 0x61, 0x74, 0xa8, 0x3a, 0xae, 0xcd, - 0x6c, 0xb4, 0x16, 0x28, 0xaa, 0xc4, 0xd1, 0x55, 0xae, 0xa8, 0x4a, 0xc5, 0xf5, 0xed, 0x91, 0xce, - 0x1e, 0xf9, 0x03, 0x55, 0xb3, 0xcd, 0xde, 0xc8, 0x1e, 0xd9, 0x3d, 0xa1, 0x3f, 0xf0, 0x1f, 0x8a, - 0x2f, 0xf1, 0x21, 0x7e, 0x05, 0x38, 0xeb, 0xdd, 0x84, 0x43, 0xcd, 0x76, 0x69, 0x6f, 0x9c, 0xf3, - 0xb5, 0xfe, 0x4e, 0xac, 0x63, 0x12, 0xed, 0x91, 0x6e, 0x51, 0x77, 0xd2, 0x73, 0xce, 0x46, 0x7c, - 0xc1, 0xeb, 0x99, 0x94, 0x91, 0x22, 0xab, 0x5e, 0x99, 0x95, 0xeb, 0x5b, 0x4c, 0x37, 0x69, 0xce, - 0xe0, 0xdd, 0xcb, 0x0c, 0x3c, 0xed, 0x11, 0x35, 0x49, 0xce, 0xee, 0xed, 0x32, 0x3b, 0x9f, 0xe9, - 0x46, 0x4f, 0xb7, 0x98, 0xc7, 0xdc, 0xac, 0x51, 0xf7, 0x5f, 0x0a, 0xa0, 0x3d, 0xdb, 0x62, 0xae, - 0x6d, 0x18, 0xd4, 0xc5, 0x74, 0xac, 0x7b, 0xba, 0x6d, 0xa1, 0xcf, 0xa1, 0xc1, 0xf7, 0x33, 0x24, - 0x8c, 0xb4, 0x95, 0x4d, 0x65, 0x6b, 0x71, 0xe7, 0x2d, 0x35, 0x3e, 0xe9, 0x08, 0x5e, 0x75, 0xce, - 0x46, 0x7c, 0xc1, 0x53, 0xb9, 0xb6, 0x3a, 0xbe, 0xad, 0x1e, 0x0e, 0xbe, 0xa0, 0x1a, 0x3b, 0xa0, - 0x8c, 0xf4, 0xd1, 0x93, 0xf3, 0x8d, 0x99, 0xe9, 0xf9, 0x06, 0xc4, 0x6b, 0x38, 0x42, 0x45, 0x87, - 0x30, 0x27, 0xd0, 0x6b, 0x02, 0x7d, 0xbb, 0x14, 0x5d, 0x6e, 0x5a, 0xc5, 0xe4, 0x57, 0xf7, 0x1f, - 0x33, 0x6a, 0xf1, 0xf0, 0xfa, 0x2f, 0x48, 0xe8, 0xb9, 0x7b, 0x84, 0x11, 0x2c, 0x80, 0xd0, 0x9b, - 0xd0, 0x70, 0x65, 0xf8, 0xed, 0xd9, 0x4d, 0x65, 0x6b, 0xb6, 0x7f, 0x43, 0x6a, 0x35, 0xc2, 0x6d, - 0xe1, 0x48, 0xa3, 0xfb, 0x44, 0x81, 0x5b, 0xf9, 0x7d, 0xef, 0xeb, 0x1e, 0x43, 0xbf, 0xc8, 0xed, - 0x5d, 0xad, 0xb6, 0x77, 0x6e, 0x2d, 0x76, 0x1e, 0x39, 0x0e, 0x57, 0x12, 0xfb, 0x3e, 0x82, 0xba, - 0xce, 0xa8, 0xe9, 0xb5, 0x6b, 0x9b, 0xb3, 0x5b, 0x8b, 0x3b, 0x6f, 0xa8, 0x25, 0x09, 0xac, 0xe6, - 0xa3, 0xeb, 0x2f, 0x49, 0xdc, 0xfa, 0x03, 0x8e, 0x80, 0x03, 0xa0, 0xee, 0x6f, 0x6b, 0x00, 0xf7, - 0xa8, 0x63, 0xd8, 0x13, 0x93, 0x5a, 0xec, 0x1a, 0xae, 0xee, 0x01, 0xcc, 0x79, 0x0e, 0xd5, 0xe4, - 0xd5, 0xbd, 0x5a, 0xba, 0x83, 0x38, 0xa8, 0x63, 0x87, 0x6a, 0xf1, 0xa5, 0xf1, 0x2f, 0x2c, 0x20, - 0xd0, 0x27, 0x30, 0xef, 0x31, 0xc2, 0x7c, 0x4f, 0x5c, 0xd9, 0xe2, 0xce, 0x6b, 0x55, 0xc0, 0x84, - 0x41, 0xbf, 0x25, 0xe1, 0xe6, 0x83, 0x6f, 0x2c, 0x81, 0xba, 0x7f, 0x9b, 0x85, 0x95, 0x58, 0x79, - 0xcf, 0xb6, 0x86, 0x3a, 0xe3, 0x29, 0x7d, 0x17, 0xe6, 0xd8, 0xc4, 0xa1, 0xe2, 0x4c, 0x9a, 0xfd, - 0x57, 0xc3, 0x60, 0x4e, 0x26, 0x0e, 0x7d, 0x76, 0xbe, 0xb1, 0x56, 0x60, 0xc2, 0x45, 0x58, 0x18, - 0xa1, 0xfd, 0x28, 0xce, 0x9a, 0x30, 0x7f, 0x27, 0xed, 0xfc, 0xd9, 0xf9, 0x46, 0x41, 0x03, 0x51, - 0x23, 0xa4, 0x74, 0x88, 0xe8, 0x0b, 0x68, 0x19, 0xc4, 0x63, 0xa7, 0xce, 0x90, 0x30, 0x7a, 0xa2, - 0x9b, 0xb4, 0x3d, 0x2f, 0x76, 0xff, 0x7a, 0xb5, 0x8b, 0xe2, 0x16, 0xfd, 0x5b, 0x32, 0x82, 0xd6, - 0x7e, 0x0a, 0x09, 0x67, 0x90, 0xd1, 0x18, 0x10, 0x5f, 0x39, 0x71, 0x89, 0xe5, 0x05, 0xbb, 0xe2, - 0xfe, 0x16, 0xae, 0xec, 0x6f, 0x5d, 0xfa, 0x43, 0xfb, 0x39, 0x34, 0x5c, 0xe0, 0x01, 0xbd, 0x02, - 0xf3, 0x2e, 0x25, 0x9e, 0x6d, 0xb5, 0xe7, 0xc4, 0x89, 0x45, 0xd7, 0x85, 0xc5, 0x2a, 0x96, 0x52, - 0xf4, 0x1a, 0x2c, 0x98, 0xd4, 0xf3, 0xc8, 0x88, 0xb6, 0xeb, 0x42, 0x71, 0x59, 0x2a, 0x2e, 0x1c, - 0x04, 0xcb, 0x38, 0x94, 0x77, 0xff, 0xa8, 0x40, 0x2b, 0xbe, 0xa6, 0x6b, 0xa8, 0xd5, 0x8f, 0xd2, - 0xb5, 0xfa, 0xfd, 0x0a, 0xc9, 0x59, 0x52, 0xa3, 0xff, 0xa8, 0x01, 0x8a, 0x95, 0xb0, 0x6d, 0x18, - 0x03, 0xa2, 0x9d, 0xa1, 0x4d, 0x98, 0xb3, 0x88, 0x19, 0xe6, 0x64, 0x54, 0x20, 0x3f, 0x21, 0x26, - 0xc5, 0x42, 0x82, 0xbe, 0x54, 0x00, 0xf9, 0xe2, 0x36, 0x87, 0xbb, 0x96, 0x65, 0x33, 0xc2, 0x0f, - 0x38, 0x0c, 0x68, 0xaf, 0x42, 0x40, 0xa1, 0x2f, 0xf5, 0x34, 0x87, 0x72, 0xdf, 0x62, 0xee, 0x24, - 0xbe, 0xd8, 0xbc, 0x02, 0x2e, 0x70, 0x8d, 0x7e, 0x0e, 0xe0, 0x4a, 0xcc, 0x13, 0x5b, 0x96, 0x6d, - 0x79, 0x0f, 0x08, 0xdd, 0xef, 0xd9, 0xd6, 0x43, 0x7d, 0x14, 0x37, 0x16, 0x1c, 0x41, 0xe0, 0x04, - 0xdc, 0xfa, 0x7d, 0x58, 0x2b, 0x89, 0x13, 0xdd, 0x80, 0xd9, 0x33, 0x3a, 0x09, 0x8e, 0x0a, 0xf3, - 0x9f, 0x68, 0x15, 0xea, 0x63, 0x62, 0xf8, 0x34, 0xa8, 0x49, 0x1c, 0x7c, 0xdc, 0xa9, 0xbd, 0xa7, - 0x74, 0x7f, 0x5f, 0x4f, 0x66, 0x0a, 0xef, 0x37, 0x68, 0x8b, 0x3f, 0x0f, 0x8e, 0xa1, 0x6b, 0xc4, - 0x13, 0x18, 0xf5, 0xfe, 0x0b, 0xc1, 0xd3, 0x10, 0xac, 0xe1, 0x48, 0x8a, 0x7e, 0x09, 0x0d, 0x8f, - 0x1a, 0x54, 0x63, 0xb6, 0x2b, 0x5b, 0xdc, 0xdb, 0x15, 0x73, 0x8a, 0x0c, 0xa8, 0x71, 0x2c, 0x4d, - 0x03, 0xf8, 0xf0, 0x0b, 0x47, 0x90, 0xe8, 0x13, 0x68, 0x30, 0x6a, 0x3a, 0x06, 0x61, 0x54, 0x9e, - 0x5e, 0x2a, 0xaf, 0x78, 0xef, 0xe0, 0x60, 0x47, 0xf6, 0xf0, 0x44, 0xaa, 0x89, 0xee, 0x19, 0xe5, - 0x69, 0xb8, 0x8a, 0x23, 0x18, 0xf4, 0x33, 0x68, 0x78, 0x8c, 0xbf, 0xea, 0xa3, 0x89, 0xa8, 0xb6, - 0x8b, 0x9e, 0x95, 0x64, 0x1f, 0x0d, 0x4c, 0x62, 0xe8, 0x70, 0x05, 0x47, 0x70, 0x68, 0x17, 0x96, - 0x4d, 0xdd, 0xc2, 0x94, 0x0c, 0x27, 0xc7, 0x54, 0xb3, 0xad, 0xa1, 0x27, 0xca, 0xb4, 0xde, 0x5f, - 0x93, 0x46, 0xcb, 0x07, 0x69, 0x31, 0xce, 0xea, 0xa3, 0x7d, 0x58, 0x0d, 0x9f, 0xdd, 0x8f, 0x74, - 0x8f, 0xd9, 0xee, 0x64, 0x5f, 0x37, 0x75, 0x26, 0x7a, 0x5e, 0xbd, 0xdf, 0x9e, 0x9e, 0x6f, 0xac, - 0xe2, 0x02, 0x39, 0x2e, 0xb4, 0xe2, 0x7d, 0xc5, 0x21, 0xbe, 0x47, 0x87, 0xa2, 0x87, 0x35, 0xe2, - 0xbe, 0x72, 0x24, 0x56, 0xb1, 0x94, 0xa2, 0x9f, 0xa6, 0xd2, 0xb4, 0x71, 0xb5, 0x34, 0x6d, 0x95, - 0xa7, 0x28, 0x3a, 0x85, 0x35, 0xc7, 0xb5, 0x47, 0x2e, 0xf5, 0xbc, 0x7b, 0x94, 0x0c, 0x0d, 0xdd, - 0xa2, 0xe1, 0xc9, 0x34, 0xc5, 0x8e, 0x5e, 0x9a, 0x9e, 0x6f, 0xac, 0x1d, 0x15, 0xab, 0xe0, 0x32, - 0xdb, 0xee, 0x5f, 0xe6, 0xe0, 0x46, 0xf6, 0x8d, 0x43, 0x1f, 0x03, 0xb2, 0x07, 0x1e, 0x75, 0xc7, - 0x74, 0xf8, 0x61, 0x30, 0xb8, 0xf1, 0xe9, 0x46, 0x11, 0xd3, 0x4d, 0x54, 0xb7, 0x87, 0x39, 0x0d, - 0x5c, 0x60, 0x15, 0xcc, 0x47, 0xb2, 0x00, 0x6a, 0x22, 0xd0, 0xc4, 0x7c, 0x94, 0x2b, 0x82, 0x5d, - 0x58, 0x96, 0xb5, 0x1f, 0x0a, 0x45, 0xb2, 0x26, 0xee, 0xfd, 0x34, 0x2d, 0xc6, 0x59, 0x7d, 0x74, - 0x17, 0x96, 0x5c, 0x9e, 0x07, 0x11, 0xc0, 0x82, 0x00, 0xf8, 0x8e, 0x04, 0x58, 0xc2, 0x49, 0x21, - 0x4e, 0xeb, 0xa2, 0x0f, 0xe1, 0x26, 0x19, 0x13, 0xdd, 0x20, 0x03, 0x83, 0x46, 0x00, 0x73, 0x02, - 0xe0, 0x45, 0x09, 0x70, 0x73, 0x37, 0xab, 0x80, 0xf3, 0x36, 0xe8, 0x00, 0x56, 0x7c, 0x2b, 0x0f, - 0x15, 0x24, 0xf1, 0x4b, 0x12, 0x6a, 0xe5, 0x34, 0xaf, 0x82, 0x8b, 0xec, 0xd0, 0xe7, 0x00, 0x5a, - 0xf8, 0xaa, 0x7b, 0xed, 0x79, 0xd1, 0x86, 0xdf, 0xac, 0x50, 0x6c, 0xd1, 0x28, 0x10, 0xb7, 0xc0, - 0x68, 0xc9, 0xc3, 0x09, 0x4c, 0x74, 0x07, 0x5a, 0x9a, 0x6d, 0x18, 0x22, 0xf3, 0xf7, 0x6c, 0xdf, - 0x62, 0x22, 0x79, 0xeb, 0x7d, 0xc4, 0x1f, 0xfb, 0xbd, 0x94, 0x04, 0x67, 0x34, 0xbb, 0x7f, 0x56, - 0x92, 0xcf, 0x4c, 0x58, 0xce, 0xe8, 0x4e, 0x6a, 0xf4, 0x79, 0x25, 0x33, 0xfa, 0xdc, 0xca, 0x5b, - 0x24, 0x26, 0x1f, 0x1d, 0x96, 0x78, 0xf2, 0xeb, 0xd6, 0x28, 0xb8, 0x70, 0xd9, 0x12, 0xdf, 0xba, - 0xb0, 0x94, 0x22, 0xed, 0xc4, 0xc3, 0x78, 0x53, 0xdc, 0x79, 0x52, 0x88, 0xd3, 0xc8, 0xdd, 0x0f, - 0xa0, 0x95, 0xae, 0xc3, 0xd4, 0x4c, 0xaf, 0x5c, 0x3a, 0xd3, 0x7f, 0xab, 0xc0, 0x5a, 0x89, 0x77, - 0x64, 0x40, 0xcb, 0x24, 0x8f, 0x13, 0xd7, 0x7c, 0xe9, 0x6c, 0xcc, 0x59, 0x93, 0x1a, 0xb0, 0x26, - 0xf5, 0x81, 0xc5, 0x0e, 0xdd, 0x63, 0xe6, 0xea, 0xd6, 0x28, 0xb8, 0x87, 0x83, 0x14, 0x16, 0xce, - 0x60, 0xa3, 0xcf, 0xa0, 0x61, 0x92, 0xc7, 0xc7, 0xbe, 0x3b, 0x2a, 0x3a, 0xaf, 0x6a, 0x7e, 0xc4, - 0xfb, 0x71, 0x20, 0x51, 0x70, 0x84, 0xd7, 0x3d, 0x84, 0xcd, 0xd4, 0x26, 0x79, 0xab, 0xa0, 0x0f, - 0x7d, 0xe3, 0x98, 0xc6, 0x17, 0xfe, 0x06, 0x34, 0x1d, 0xe2, 0x32, 0x3d, 0x6a, 0x17, 0xf5, 0xfe, - 0xd2, 0xf4, 0x7c, 0xa3, 0x79, 0x14, 0x2e, 0xe2, 0x58, 0xde, 0xfd, 0xb7, 0x02, 0xf5, 0x63, 0x8d, - 0x18, 0xf4, 0x1a, 0xa8, 0xc3, 0xbd, 0x14, 0x75, 0xe8, 0x96, 0x26, 0x91, 0x88, 0xa7, 0x94, 0x35, - 0xec, 0x67, 0x58, 0xc3, 0xcb, 0x97, 0xe0, 0x5c, 0x4c, 0x18, 0xde, 0x87, 0x66, 0xe4, 0x2e, 0xd5, - 0x25, 0x95, 0xcb, 0xba, 0x64, 0xf7, 0x77, 0x35, 0x58, 0x4c, 0xb8, 0xb8, 0x9a, 0x35, 0x3f, 0xee, - 0xc4, 0xa0, 0xc1, 0x3b, 0xc9, 0x4e, 0x95, 0x8d, 0xa8, 0xe1, 0x50, 0x11, 0xcc, 0x6f, 0xf1, 0xeb, - 0x9d, 0x9f, 0x35, 0x3e, 0x80, 0x16, 0x23, 0xee, 0x88, 0xb2, 0x50, 0x26, 0x0e, 0xac, 0x19, 0x93, - 0x87, 0x93, 0x94, 0x14, 0x67, 0xb4, 0xd7, 0xef, 0xc2, 0x52, 0xca, 0xd9, 0x95, 0x86, 0xb0, 0x2f, - 0xf9, 0xe1, 0xc4, 0xc9, 0x79, 0x0d, 0xd9, 0xf5, 0x71, 0x2a, 0xbb, 0xb6, 0xca, 0x0f, 0x33, 0x51, - 0x32, 0x65, 0x39, 0x86, 0x33, 0x39, 0xf6, 0x7a, 0x25, 0xb4, 0x8b, 0x33, 0xed, 0x9f, 0x35, 0x58, - 0x4d, 0x68, 0xc7, 0xdc, 0xf4, 0x87, 0xa9, 0x06, 0xbd, 0x95, 0x69, 0xd0, 0xed, 0x22, 0x9b, 0xe7, - 0x46, 0x4e, 0x8b, 0x09, 0xe3, 0xec, 0xff, 0x22, 0x61, 0xfc, 0x93, 0x02, 0xcb, 0x89, 0xb3, 0xbb, - 0x06, 0xc6, 0xf8, 0x20, 0xcd, 0x18, 0x5f, 0xae, 0x92, 0x34, 0x25, 0x94, 0xf1, 0xab, 0xf9, 0x54, - 0xf0, 0xff, 0xf7, 0x24, 0xe6, 0xd7, 0xb0, 0x3a, 0xb6, 0x0d, 0xdf, 0xa4, 0x7b, 0x06, 0xd1, 0xcd, - 0x50, 0x81, 0x0f, 0x7d, 0xb3, 0xd9, 0x3f, 0x86, 0x22, 0x78, 0xea, 0x7a, 0xba, 0xc7, 0xa8, 0xc5, - 0x3e, 0x8d, 0x2d, 0xfb, 0xdf, 0x95, 0x4e, 0x56, 0x3f, 0x2d, 0x80, 0xc3, 0x85, 0x4e, 0xd0, 0x0f, - 0x60, 0x91, 0x0f, 0xcc, 0xba, 0x46, 0x39, 0xf7, 0x96, 0x89, 0xb5, 0x22, 0x81, 0x16, 0x8f, 0x63, - 0x11, 0x4e, 0xea, 0xa1, 0x47, 0xb0, 0xe2, 0xd8, 0xc3, 0x03, 0x62, 0x91, 0x11, 0xe5, 0x63, 0xc6, - 0x91, 0x6d, 0xe8, 0xda, 0x44, 0x30, 0x9b, 0x66, 0xff, 0xdd, 0x70, 0xb8, 0x3c, 0xca, 0xab, 0x3c, - 0xe3, 0x14, 0x21, 0xbf, 0x2c, 0x8a, 0xba, 0x08, 0x12, 0xb9, 0xd0, 0xf2, 0xe5, 0x73, 0x2f, 0x89, - 0x5e, 0xf0, 0x17, 0xce, 0x4e, 0x95, 0x0c, 0x3b, 0x4d, 0x59, 0xc6, 0xdd, 0x3f, 0xbd, 0x8e, 0x33, - 0x1e, 0x4a, 0x89, 0x5b, 0xe3, 0xbf, 0x22, 0x6e, 0x05, 0x4c, 0xb2, 0x79, 0x35, 0x26, 0xd9, 0xfd, - 0x43, 0x1d, 0x6e, 0xe6, 0xba, 0x2d, 0xfa, 0xf1, 0x05, 0x24, 0xe9, 0xd6, 0x73, 0x23, 0x48, 0x39, - 0x76, 0x33, 0x7b, 0x05, 0x76, 0xb3, 0x0b, 0xcb, 0x9a, 0xef, 0xba, 0xd4, 0x62, 0x19, 0x6e, 0x13, - 0x9d, 0xc5, 0x5e, 0x5a, 0x8c, 0xb3, 0xfa, 0x45, 0x04, 0xad, 0x7e, 0x45, 0x82, 0x96, 0x8c, 0x42, - 0x0e, 0xd9, 0x41, 0xe6, 0xe6, 0xa3, 0x90, 0xb3, 0x76, 0x56, 0x9f, 0x0f, 0x18, 0x01, 0x6a, 0x84, - 0xb0, 0x90, 0x1e, 0x30, 0x4e, 0x53, 0x52, 0x9c, 0xd1, 0x2e, 0x20, 0x3b, 0xcd, 0xaa, 0x64, 0x07, - 0x91, 0x14, 0x15, 0x03, 0xd1, 0x26, 0xb6, 0xab, 0x94, 0x43, 0x75, 0x2e, 0x56, 0xc8, 0x42, 0x17, - 0xaf, 0xce, 0x42, 0xbb, 0x7f, 0x55, 0xe0, 0xc5, 0xd2, 0x82, 0x44, 0xbb, 0xa9, 0xe7, 0x7f, 0x3b, - 0xf3, 0xfc, 0x7f, 0xaf, 0xd4, 0x30, 0x31, 0x03, 0xb8, 0xc5, 0x34, 0xed, 0xfd, 0x6a, 0x34, 0xad, - 0x80, 0x43, 0x5c, 0xce, 0xd7, 0xfa, 0xdb, 0x4f, 0x9e, 0x76, 0x66, 0xbe, 0x7e, 0xda, 0x99, 0xf9, - 0xe6, 0x69, 0x67, 0xe6, 0x37, 0xd3, 0x8e, 0xf2, 0x64, 0xda, 0x51, 0xbe, 0x9e, 0x76, 0x94, 0x6f, - 0xa6, 0x1d, 0xe5, 0xef, 0xd3, 0x8e, 0xf2, 0xd5, 0xb7, 0x9d, 0x99, 0xcf, 0x16, 0xa4, 0xc7, 0xff, - 0x04, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x02, 0xfa, 0x27, 0xed, 0x1b, 0x00, 0x00, + // 1968 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0xd7, 0x4a, 0xa2, 0x44, 0x3d, 0x45, 0x94, 0x3d, 0x52, 0x2d, 0x46, 0x69, 0x49, 0x81, 0x35, + 0x12, 0xe5, 0x43, 0xcb, 0x58, 0x49, 0x83, 0xc4, 0x2e, 0xdc, 0x8a, 0x92, 0x1b, 0x3b, 0x90, 0x22, + 0x65, 0x24, 0xc5, 0x68, 0xfa, 0x81, 0x0c, 0xc9, 0x31, 0xb5, 0xd1, 0x7e, 0x61, 0x77, 0xc8, 0x98, + 0xe8, 0xa5, 0x7f, 0x40, 0x81, 0xf4, 0xdc, 0xbf, 0xa2, 0xb7, 0x16, 0xed, 0xad, 0x87, 0xc2, 0xc7, + 0xa0, 0x97, 0xa6, 0x17, 0xa2, 0x66, 0xae, 0xed, 0xad, 0xbd, 0x18, 0x28, 0x50, 0xcc, 0xec, 0xec, + 0xf7, 0xae, 0xb4, 0x2c, 0x60, 0x01, 0xcd, 0x8d, 0x3b, 0xef, 0xbd, 0xdf, 0x9b, 0x99, 0xf7, 0x31, + 0xef, 0x47, 0xf8, 0xe1, 0xf9, 0xbb, 0xae, 0xaa, 0x59, 0xcd, 0xf3, 0x7e, 0x9b, 0x3a, 0x26, 0x65, + 0xd4, 0x6d, 0x0e, 0xa8, 0xd9, 0xb5, 0x9c, 0xa6, 0x14, 0x10, 0x5b, 0x6b, 0x12, 0xdb, 0x76, 0x9b, + 0x83, 0x5b, 0x6d, 0xca, 0xc8, 0xad, 0x66, 0x8f, 0x9a, 0xd4, 0x21, 0x8c, 0x76, 0x55, 0xdb, 0xb1, + 0x98, 0x85, 0xd6, 0x3c, 0x45, 0x95, 0xd8, 0x9a, 0xca, 0x15, 0x55, 0xa9, 0xb8, 0xbe, 0xd5, 0xd3, + 0xd8, 0x59, 0xbf, 0xad, 0x76, 0x2c, 0xa3, 0xd9, 0xb3, 0x7a, 0x56, 0x53, 0xe8, 0xb7, 0xfb, 0x8f, + 0xc4, 0x97, 0xf8, 0x10, 0xbf, 0x3c, 0x9c, 0xf5, 0x46, 0xc4, 0x61, 0xc7, 0x72, 0x68, 0x73, 0x90, + 0xf2, 0xb5, 0xfe, 0x76, 0xa8, 0x63, 0x90, 0xce, 0x99, 0x66, 0x52, 0x67, 0xd8, 0xb4, 0xcf, 0x7b, + 0x7c, 0xc1, 0x6d, 0x1a, 0x94, 0x91, 0x2c, 0xab, 0x66, 0x9e, 0x95, 0xd3, 0x37, 0x99, 0x66, 0xd0, + 0x94, 0xc1, 0x3b, 0x97, 0x19, 0xb8, 0x9d, 0x33, 0x6a, 0x90, 0x94, 0xdd, 0x5b, 0x79, 0x76, 0x7d, + 0xa6, 0xe9, 0x4d, 0xcd, 0x64, 0x2e, 0x73, 0x92, 0x46, 0x8d, 0x7f, 0x2b, 0x80, 0x76, 0x2d, 0x93, + 0x39, 0x96, 0xae, 0x53, 0x07, 0xd3, 0x81, 0xe6, 0x6a, 0x96, 0x89, 0x3e, 0x85, 0x32, 0x3f, 0x4f, + 0x97, 0x30, 0x52, 0x55, 0x36, 0x94, 0xcd, 0xc5, 0xed, 0x37, 0xd5, 0xf0, 0xa6, 0x03, 0x78, 0xd5, + 0x3e, 0xef, 0xf1, 0x05, 0x57, 0xe5, 0xda, 0xea, 0xe0, 0x96, 0x7a, 0xd8, 0xfe, 0x8c, 0x76, 0xd8, + 0x01, 0x65, 0xa4, 0x85, 0x9e, 0x8c, 0xea, 0x53, 0xe3, 0x51, 0x1d, 0xc2, 0x35, 0x1c, 0xa0, 0xa2, + 0x43, 0x98, 0x15, 0xe8, 0xd3, 0x02, 0x7d, 0x2b, 0x17, 0x5d, 0x1e, 0x5a, 0xc5, 0xe4, 0xf3, 0x7b, + 0x8f, 0x19, 0x35, 0xf9, 0xf6, 0x5a, 0x2f, 0x48, 0xe8, 0xd9, 0x3d, 0xc2, 0x08, 0x16, 0x40, 0xe8, + 0x0d, 0x28, 0x3b, 0x72, 0xfb, 0xd5, 0x99, 0x0d, 0x65, 0x73, 0xa6, 0x75, 0x4d, 0x6a, 0x95, 0xfd, + 0x63, 0xe1, 0x40, 0xa3, 0xf1, 0x44, 0x81, 0x1b, 0xe9, 0x73, 0xef, 0x6b, 0x2e, 0x43, 0x3f, 0x4d, + 0x9d, 0x5d, 0x2d, 0x76, 0x76, 0x6e, 0x2d, 0x4e, 0x1e, 0x38, 0xf6, 0x57, 0x22, 0xe7, 0x3e, 0x82, + 0x92, 0xc6, 0xa8, 0xe1, 0x56, 0xa7, 0x37, 0x66, 0x36, 0x17, 0xb7, 0x5f, 0x57, 0x73, 0x12, 0x58, + 0x4d, 0xef, 0xae, 0xb5, 0x24, 0x71, 0x4b, 0x0f, 0x38, 0x02, 0xf6, 0x80, 0x1a, 0xbf, 0x9a, 0x06, + 0xd8, 0xa3, 0xb6, 0x6e, 0x0d, 0x0d, 0x6a, 0xb2, 0x2b, 0x08, 0xdd, 0x03, 0x98, 0x75, 0x6d, 0xda, + 0x91, 0xa1, 0x7b, 0x25, 0xf7, 0x04, 0xe1, 0xa6, 0x8e, 0x6d, 0xda, 0x09, 0x83, 0xc6, 0xbf, 0xb0, + 0x80, 0x40, 0x1f, 0xc1, 0x9c, 0xcb, 0x08, 0xeb, 0xbb, 0x22, 0x64, 0x8b, 0xdb, 0xaf, 0x16, 0x01, + 0x13, 0x06, 0xad, 0x8a, 0x84, 0x9b, 0xf3, 0xbe, 0xb1, 0x04, 0x6a, 0xfc, 0x75, 0x06, 0x56, 0x42, + 0xe5, 0x5d, 0xcb, 0xec, 0x6a, 0x8c, 0xa7, 0xf4, 0x1d, 0x98, 0x65, 0x43, 0x9b, 0x8a, 0x3b, 0x59, + 0x68, 0xbd, 0xe2, 0x6f, 0xe6, 0x64, 0x68, 0xd3, 0x67, 0xa3, 0xfa, 0x5a, 0x86, 0x09, 0x17, 0x61, + 0x61, 0x84, 0xf6, 0x83, 0x7d, 0x4e, 0x0b, 0xf3, 0xb7, 0xe3, 0xce, 0x9f, 0x8d, 0xea, 0x19, 0x0d, + 0x44, 0x0d, 0x90, 0xe2, 0x5b, 0x44, 0x9f, 0x41, 0x45, 0x27, 0x2e, 0x3b, 0xb5, 0xbb, 0x84, 0xd1, + 0x13, 0xcd, 0xa0, 0xd5, 0x39, 0x71, 0xfa, 0xd7, 0x8a, 0x05, 0x8a, 0x5b, 0xb4, 0x6e, 0xc8, 0x1d, + 0x54, 0xf6, 0x63, 0x48, 0x38, 0x81, 0x8c, 0x06, 0x80, 0xf8, 0xca, 0x89, 0x43, 0x4c, 0xd7, 0x3b, + 0x15, 0xf7, 0x37, 0x3f, 0xb1, 0xbf, 0x75, 0xe9, 0x0f, 0xed, 0xa7, 0xd0, 0x70, 0x86, 0x07, 0xf4, + 0x32, 0xcc, 0x39, 0x94, 0xb8, 0x96, 0x59, 0x9d, 0x15, 0x37, 0x16, 0x84, 0x0b, 0x8b, 0x55, 0x2c, + 0xa5, 0xe8, 0x55, 0x98, 0x37, 0xa8, 0xeb, 0x92, 0x1e, 0xad, 0x96, 0x84, 0xe2, 0xb2, 0x54, 0x9c, + 0x3f, 0xf0, 0x96, 0xb1, 0x2f, 0x6f, 0xfc, 0x5e, 0x81, 0x4a, 0x18, 0xa6, 0x2b, 0xa8, 0xd5, 0xfb, + 0xf1, 0x5a, 0xfd, 0x6e, 0x81, 0xe4, 0xcc, 0xa9, 0xd1, 0x7f, 0x4c, 0x03, 0x0a, 0x95, 0xb0, 0xa5, + 0xeb, 0x6d, 0xd2, 0x39, 0x47, 0x1b, 0x30, 0x6b, 0x12, 0xc3, 0xcf, 0xc9, 0xa0, 0x40, 0x3e, 0x24, + 0x06, 0xc5, 0x42, 0x82, 0xbe, 0x50, 0x00, 0xf5, 0x45, 0x34, 0xbb, 0x3b, 0xa6, 0x69, 0x31, 0xc2, + 0x2f, 0xd8, 0xdf, 0xd0, 0x6e, 0x81, 0x0d, 0xf9, 0xbe, 0xd4, 0xd3, 0x14, 0xca, 0x3d, 0x93, 0x39, + 0xc3, 0x30, 0xb0, 0x69, 0x05, 0x9c, 0xe1, 0x1a, 0xfd, 0x04, 0xc0, 0x91, 0x98, 0x27, 0x96, 0x2c, + 0xdb, 0xfc, 0x1e, 0xe0, 0xbb, 0xdf, 0xb5, 0xcc, 0x47, 0x5a, 0x2f, 0x6c, 0x2c, 0x38, 0x80, 0xc0, + 0x11, 0xb8, 0xf5, 0x7b, 0xb0, 0x96, 0xb3, 0x4f, 0x74, 0x0d, 0x66, 0xce, 0xe9, 0xd0, 0xbb, 0x2a, + 0xcc, 0x7f, 0xa2, 0x55, 0x28, 0x0d, 0x88, 0xde, 0xa7, 0x5e, 0x4d, 0x62, 0xef, 0xe3, 0xf6, 0xf4, + 0xbb, 0x4a, 0xe3, 0xb7, 0xa5, 0x68, 0xa6, 0xf0, 0x7e, 0x83, 0x36, 0xf9, 0xf3, 0x60, 0xeb, 0x5a, + 0x87, 0xb8, 0x02, 0xa3, 0xd4, 0x7a, 0xc1, 0x7b, 0x1a, 0xbc, 0x35, 0x1c, 0x48, 0xd1, 0xcf, 0xa0, + 0xec, 0x52, 0x9d, 0x76, 0x98, 0xe5, 0xc8, 0x16, 0xf7, 0x56, 0xc1, 0x9c, 0x22, 0x6d, 0xaa, 0x1f, + 0x4b, 0x53, 0x0f, 0xde, 0xff, 0xc2, 0x01, 0x24, 0xfa, 0x08, 0xca, 0x8c, 0x1a, 0xb6, 0x4e, 0x18, + 0x95, 0xb7, 0x17, 0xcb, 0x2b, 0xde, 0x3b, 0x38, 0xd8, 0x91, 0xd5, 0x3d, 0x91, 0x6a, 0xa2, 0x7b, + 0x06, 0x79, 0xea, 0xaf, 0xe2, 0x00, 0x06, 0xfd, 0x18, 0xca, 0x2e, 0xe3, 0xaf, 0x7a, 0x6f, 0x28, + 0xaa, 0xed, 0xa2, 0x67, 0x25, 0xda, 0x47, 0x3d, 0x93, 0x10, 0xda, 0x5f, 0xc1, 0x01, 0x1c, 0xda, + 0x81, 0x65, 0x43, 0x33, 0x31, 0x25, 0xdd, 0xe1, 0x31, 0xed, 0x58, 0x66, 0xd7, 0x15, 0x65, 0x5a, + 0x6a, 0xad, 0x49, 0xa3, 0xe5, 0x83, 0xb8, 0x18, 0x27, 0xf5, 0xd1, 0x3e, 0xac, 0xfa, 0xcf, 0xee, + 0x7d, 0xcd, 0x65, 0x96, 0x33, 0xdc, 0xd7, 0x0c, 0x8d, 0x89, 0x9e, 0x57, 0x6a, 0x55, 0xc7, 0xa3, + 0xfa, 0x2a, 0xce, 0x90, 0xe3, 0x4c, 0x2b, 0xde, 0x57, 0x6c, 0xd2, 0x77, 0x69, 0x57, 0xf4, 0xb0, + 0x72, 0xd8, 0x57, 0x8e, 0xc4, 0x2a, 0x96, 0x52, 0xf4, 0x30, 0x96, 0xa6, 0xe5, 0xc9, 0xd2, 0xb4, + 0x92, 0x9f, 0xa2, 0xe8, 0x14, 0xd6, 0x6c, 0xc7, 0xea, 0x39, 0xd4, 0x75, 0xf7, 0x28, 0xe9, 0xea, + 0x9a, 0x49, 0xfd, 0x9b, 0x59, 0x10, 0x27, 0x7a, 0x69, 0x3c, 0xaa, 0xaf, 0x1d, 0x65, 0xab, 0xe0, + 0x3c, 0xdb, 0xc6, 0x9f, 0x66, 0xe1, 0x5a, 0xf2, 0x8d, 0x43, 0x1f, 0x00, 0xb2, 0xda, 0x2e, 0x75, + 0x06, 0xb4, 0xfb, 0xbe, 0x37, 0xb8, 0xf1, 0xe9, 0x46, 0x11, 0xd3, 0x4d, 0x50, 0xb7, 0x87, 0x29, + 0x0d, 0x9c, 0x61, 0xe5, 0xcd, 0x47, 0xb2, 0x00, 0xa6, 0xc5, 0x46, 0x23, 0xf3, 0x51, 0xaa, 0x08, + 0x76, 0x60, 0x59, 0xd6, 0xbe, 0x2f, 0x14, 0xc9, 0x1a, 0x89, 0xfb, 0x69, 0x5c, 0x8c, 0x93, 0xfa, + 0xe8, 0x0e, 0x2c, 0x39, 0x3c, 0x0f, 0x02, 0x80, 0x79, 0x01, 0xf0, 0x2d, 0x09, 0xb0, 0x84, 0xa3, + 0x42, 0x1c, 0xd7, 0x45, 0xef, 0xc3, 0x75, 0x32, 0x20, 0x9a, 0x4e, 0xda, 0x3a, 0x0d, 0x00, 0x66, + 0x05, 0xc0, 0x8b, 0x12, 0xe0, 0xfa, 0x4e, 0x52, 0x01, 0xa7, 0x6d, 0xd0, 0x01, 0xac, 0xf4, 0xcd, + 0x34, 0x94, 0x97, 0xc4, 0x2f, 0x49, 0xa8, 0x95, 0xd3, 0xb4, 0x0a, 0xce, 0xb2, 0x43, 0x9f, 0x02, + 0x74, 0xfc, 0x57, 0xdd, 0xad, 0xce, 0x89, 0x36, 0xfc, 0x46, 0x81, 0x62, 0x0b, 0x46, 0x81, 0xb0, + 0x05, 0x06, 0x4b, 0x2e, 0x8e, 0x60, 0xa2, 0xdb, 0x50, 0xe9, 0x58, 0xba, 0x2e, 0x32, 0x7f, 0xd7, + 0xea, 0x9b, 0x4c, 0x24, 0x6f, 0xa9, 0x85, 0xf8, 0x63, 0xbf, 0x1b, 0x93, 0xe0, 0x84, 0x66, 0xe3, + 0x8f, 0x4a, 0xf4, 0x99, 0xf1, 0xcb, 0x19, 0xdd, 0x8e, 0x8d, 0x3e, 0x2f, 0x27, 0x46, 0x9f, 0x1b, + 0x69, 0x8b, 0xc8, 0xe4, 0xa3, 0xc1, 0x12, 0x4f, 0x7e, 0xcd, 0xec, 0x79, 0x01, 0x97, 0x2d, 0xf1, + 0xcd, 0x0b, 0x4b, 0x29, 0xd0, 0x8e, 0x3c, 0x8c, 0xd7, 0x45, 0xcc, 0xa3, 0x42, 0x1c, 0x47, 0x6e, + 0xdc, 0x85, 0x4a, 0xbc, 0x0e, 0x63, 0x33, 0xbd, 0x72, 0xe9, 0x4c, 0xff, 0xb5, 0x02, 0x6b, 0x39, + 0xde, 0x91, 0x0e, 0x15, 0x83, 0x3c, 0x8e, 0x84, 0xf9, 0xd2, 0xd9, 0x98, 0xb3, 0x26, 0xd5, 0x63, + 0x4d, 0xea, 0x03, 0x93, 0x1d, 0x3a, 0xc7, 0xcc, 0xd1, 0xcc, 0x9e, 0x17, 0x87, 0x83, 0x18, 0x16, + 0x4e, 0x60, 0xa3, 0x4f, 0xa0, 0x6c, 0x90, 0xc7, 0xc7, 0x7d, 0xa7, 0x97, 0x75, 0x5f, 0xc5, 0xfc, + 0x88, 0xf7, 0xe3, 0x40, 0xa2, 0xe0, 0x00, 0xaf, 0x71, 0x08, 0x1b, 0xb1, 0x43, 0xf2, 0x56, 0x41, + 0x1f, 0xf5, 0xf5, 0x63, 0x1a, 0x06, 0xfc, 0x75, 0x58, 0xb0, 0x89, 0xc3, 0xb4, 0xa0, 0x5d, 0x94, + 0x5a, 0x4b, 0xe3, 0x51, 0x7d, 0xe1, 0xc8, 0x5f, 0xc4, 0xa1, 0xbc, 0xf1, 0x1f, 0x05, 0x4a, 0xc7, + 0x1d, 0xa2, 0xd3, 0x2b, 0xa0, 0x0e, 0x7b, 0x31, 0xea, 0xd0, 0xc8, 0x4d, 0x22, 0xb1, 0x9f, 0x5c, + 0xd6, 0xb0, 0x9f, 0x60, 0x0d, 0x37, 0x2f, 0xc1, 0xb9, 0x98, 0x30, 0xbc, 0x07, 0x0b, 0x81, 0xbb, + 0x58, 0x97, 0x54, 0x2e, 0xeb, 0x92, 0x8d, 0xdf, 0x4c, 0xc3, 0x62, 0xc4, 0xc5, 0x64, 0xd6, 0xfc, + 0xba, 0x23, 0x83, 0x06, 0xef, 0x24, 0xdb, 0x45, 0x0e, 0xa2, 0xfa, 0x43, 0x85, 0x37, 0xbf, 0x85, + 0xaf, 0x77, 0x7a, 0xd6, 0xb8, 0x0b, 0x15, 0x46, 0x9c, 0x1e, 0x65, 0xbe, 0x4c, 0x5c, 0xd8, 0x42, + 0x48, 0x1e, 0x4e, 0x62, 0x52, 0x9c, 0xd0, 0x5e, 0xbf, 0x03, 0x4b, 0x31, 0x67, 0x13, 0x0d, 0x61, + 0x5f, 0xf0, 0xcb, 0x09, 0x93, 0xf3, 0x0a, 0xb2, 0xeb, 0x83, 0x58, 0x76, 0x6d, 0xe6, 0x5f, 0x66, + 0xa4, 0x64, 0xf2, 0x72, 0x0c, 0x27, 0x72, 0xec, 0xb5, 0x42, 0x68, 0x17, 0x67, 0xda, 0x3f, 0xa7, + 0x61, 0x35, 0xa2, 0x1d, 0x72, 0xd3, 0xef, 0xc7, 0x1a, 0xf4, 0x66, 0xa2, 0x41, 0x57, 0xb3, 0x6c, + 0x9e, 0x1b, 0x39, 0xcd, 0x26, 0x8c, 0x33, 0xff, 0x8f, 0x84, 0xf1, 0x0f, 0x0a, 0x2c, 0x47, 0xee, + 0xee, 0x0a, 0x18, 0xe3, 0x83, 0x38, 0x63, 0xbc, 0x59, 0x24, 0x69, 0x72, 0x28, 0xe3, 0xbf, 0x14, + 0x68, 0x46, 0xb4, 0x8e, 0xa8, 0xe3, 0x6a, 0x2e, 0xa3, 0x26, 0xfb, 0xd8, 0xd2, 0xfb, 0x06, 0xdd, + 0xd5, 0x89, 0x66, 0x60, 0xca, 0x17, 0x34, 0xcb, 0x3c, 0xb2, 0x74, 0xad, 0x33, 0x44, 0x04, 0x16, + 0x3f, 0x3f, 0xa3, 0xe6, 0x1e, 0xd5, 0x29, 0xa3, 0x5d, 0x99, 0x4e, 0x3f, 0x90, 0xf0, 0x8b, 0x0f, + 0x43, 0xd1, 0xb3, 0x51, 0x7d, 0xb3, 0x08, 0xa2, 0xc8, 0xb2, 0x28, 0x26, 0xfa, 0x39, 0x00, 0xff, + 0x14, 0xfd, 0xa8, 0x2b, 0x13, 0xee, 0xae, 0x5f, 0x95, 0x0f, 0x03, 0xc9, 0x44, 0x0e, 0x22, 0x88, + 0x8d, 0xbf, 0xcd, 0xc7, 0x62, 0xf6, 0x8d, 0xe7, 0x6e, 0xbf, 0x80, 0xd5, 0x41, 0x78, 0x3b, 0xbe, + 0x02, 0x9f, 0x75, 0x67, 0x92, 0xff, 0x87, 0x05, 0xf0, 0x59, 0xf7, 0xda, 0xfa, 0xb6, 0x74, 0xb2, + 0xfa, 0x71, 0x06, 0x1c, 0xce, 0x74, 0x82, 0xbe, 0x07, 0x8b, 0x9c, 0x27, 0x68, 0x1d, 0xfa, 0x21, + 0x31, 0xfc, 0x7a, 0x5a, 0xf1, 0xf3, 0xe5, 0x38, 0x14, 0xe1, 0xa8, 0x1e, 0x3a, 0x83, 0x15, 0xdb, + 0xea, 0x1e, 0x10, 0x93, 0xf4, 0x28, 0x9f, 0xae, 0xbc, 0x50, 0x0a, 0x42, 0xb7, 0xd0, 0x7a, 0xc7, + 0x9f, 0xa9, 0x8f, 0xd2, 0x2a, 0xcf, 0x38, 0x33, 0x4a, 0x2f, 0x8b, 0x24, 0xc8, 0x82, 0x44, 0x0e, + 0x54, 0xfa, 0x72, 0xca, 0x91, 0xfc, 0xd6, 0xfb, 0xe7, 0x6a, 0xbb, 0x48, 0x61, 0x9d, 0xc6, 0x2c, + 0xc3, 0x47, 0x2f, 0xbe, 0x8e, 0x13, 0x1e, 0x72, 0xf9, 0x6a, 0xf9, 0x7f, 0xe2, 0xab, 0x19, 0x04, + 0x7a, 0x61, 0x42, 0x02, 0xfd, 0x67, 0x05, 0x6e, 0xda, 0x05, 0x6a, 0xa9, 0x0a, 0xe2, 0x6e, 0xee, + 0x17, 0xb9, 0x9b, 0x22, 0xb5, 0xd9, 0xda, 0x1c, 0x8f, 0xea, 0x37, 0x8b, 0x68, 0xe2, 0x42, 0xfb, + 0x6b, 0xfc, 0xae, 0x04, 0xd7, 0x53, 0xaf, 0x25, 0xfa, 0xd1, 0x05, 0x24, 0xf7, 0xc6, 0x73, 0x23, + 0xb8, 0x29, 0x76, 0x3a, 0x33, 0x01, 0x3b, 0xdd, 0x81, 0xe5, 0x4e, 0xdf, 0x71, 0xa8, 0xc9, 0x12, + 0xdc, 0x34, 0x08, 0xea, 0x6e, 0x5c, 0x8c, 0x93, 0xfa, 0x59, 0x04, 0xbb, 0x34, 0x21, 0xc1, 0x8e, + 0xee, 0x42, 0x92, 0x24, 0xaf, 0x04, 0xd3, 0xbb, 0x90, 0x5c, 0x29, 0xa9, 0xcf, 0x07, 0x44, 0x0f, + 0x35, 0x40, 0x98, 0x8f, 0x0f, 0x88, 0xa7, 0x31, 0x29, 0x4e, 0x68, 0x67, 0x90, 0xd5, 0x85, 0xa2, + 0x64, 0x15, 0x91, 0x18, 0x95, 0x06, 0xd1, 0xef, 0xb6, 0x8a, 0xe4, 0x6e, 0x71, 0x2e, 0x9d, 0xf9, + 0x2f, 0xc2, 0xe2, 0xe4, 0xff, 0x22, 0x34, 0xfe, 0xa2, 0xc0, 0x8b, 0xb9, 0x9d, 0x05, 0xed, 0xc4, + 0xc6, 0xb7, 0xad, 0xc4, 0xf8, 0xf6, 0x9d, 0x5c, 0xc3, 0xc8, 0x0c, 0xe7, 0x64, 0xd3, 0xec, 0xf7, + 0x8a, 0xd1, 0xec, 0x0c, 0x0e, 0x78, 0x39, 0xdf, 0x6e, 0x6d, 0x3d, 0x79, 0x5a, 0x9b, 0xfa, 0xf2, + 0x69, 0x6d, 0xea, 0xab, 0xa7, 0xb5, 0xa9, 0x5f, 0x8e, 0x6b, 0xca, 0x93, 0x71, 0x4d, 0xf9, 0x72, + 0x5c, 0x53, 0xbe, 0x1a, 0xd7, 0x94, 0xbf, 0x8f, 0x6b, 0xca, 0xaf, 0xbf, 0xae, 0x4d, 0x7d, 0x32, + 0x2f, 0x3d, 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0x3d, 0xfc, 0xe0, 0xc3, 0xad, 0x1d, 0x00, 0x00, } func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { @@ -1687,6 +1724,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i -= len(m.WhenScaled) + copy(dAtA[i:], m.WhenScaled) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled))) + i-- + dAtA[i] = 0x12 + i -= len(m.WhenDeleted) + copy(dAtA[i:], m.WhenDeleted) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1707,6 +1777,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.PersistentVolumeClaimRetentionPolicy != nil { + { + size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i-- dAtA[i] = 0x48 @@ -2213,6 +2295,19 @@ func (m *StatefulSetList) Size() (n int) { return n } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.WhenDeleted) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.WhenScaled) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *StatefulSetSpec) Size() (n int) { if m == nil { return 0 @@ -2244,6 +2339,10 @@ func (m *StatefulSetSpec) Size() (n int) { n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) } n += 1 + sovGenerated(uint64(m.MinReadySeconds)) + if m.PersistentVolumeClaimRetentionPolicy != nil { + l = m.PersistentVolumeClaimRetentionPolicy.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -2559,6 +2658,17 @@ func (this *StatefulSetList) String() string { }, "") return s } +func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`, + `WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`, + `WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`, + `}`, + }, "") + return s +} func (this *StatefulSetSpec) String() string { if this == nil { return "nil" @@ -2578,6 +2688,7 @@ func (this *StatefulSetSpec) String() string { `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, + `PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`, `}`, }, "") return s @@ -5436,6 +5547,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error { } return nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5724,6 +5949,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PersistentVolumeClaimRetentionPolicy == nil { + m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/apps/v1beta1/generated.proto b/staging/src/k8s.io/api/apps/v1beta1/generated.proto index 2432b6e63449..9f9df98fce6f 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/apps/v1beta1/generated.proto @@ -367,6 +367,23 @@ message StatefulSetList { repeated StatefulSet items = 2; } +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +message StatefulSetPersistentVolumeClaimRetentionPolicy { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + optional string whenDeleted = 1; + + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + optional string whenScaled = 2; +} + // A StatefulSetSpec is the specification of a StatefulSet. message StatefulSetSpec { // replicas is the desired number of replicas of the given Template. @@ -434,6 +451,12 @@ message StatefulSetSpec { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional optional int32 minReadySeconds = 9; + + // PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from + // the StatefulSet VolumeClaimTemplates. This requires the + // StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. + // +optional + optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10; } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1beta1/types.go b/staging/src/k8s.io/api/apps/v1beta1/types.go index dc9cf1e36528..832ef34f45f6 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/types.go +++ b/staging/src/k8s.io/api/apps/v1beta1/types.go @@ -158,6 +158,40 @@ type RollingUpdateStatefulSetStrategy struct { Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` } +// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine +// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is +// deleted or scaled down. +type PersistentVolumeClaimRetentionPolicyType string + +const ( + // RetainPersistentVolumeClaimRetentionPolicyType is the default + // PersistentVolumeClaimRetentionPolicy and specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will not be deleted. + RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain" + // RetentionPersistentVolumeClaimRetentionPolicyType specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will be deleted in the scenario specified in + // StatefulSetPersistentVolumeClaimRetentionPolicy. + RetentionPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +type StatefulSetPersistentVolumeClaimRetentionPolicy struct { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"` + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"` +} + // A StatefulSetSpec is the specification of a StatefulSet. type StatefulSetSpec struct { // replicas is the desired number of replicas of the given Template. @@ -225,6 +259,12 @@ type StatefulSetSpec struct { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` + + // PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from + // the StatefulSet VolumeClaimTemplates. This requires the + // StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. + // +optional + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"` } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/apps/v1beta1/types_swagger_doc_generated.go index 8ebd640095c8..e92881a35da1 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/apps/v1beta1/types_swagger_doc_generated.go @@ -227,17 +227,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string { return map_StatefulSetList } +var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{ + "": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.", + "whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.", + "whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.", +} + +func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string { + return map_StatefulSetPersistentVolumeClaimRetentionPolicy +} + var map_StatefulSetSpec = map[string]string{ - "": "A StatefulSetSpec is the specification of a StatefulSet.", - "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", - "selector": "selector is a label query over pods that should match the replica count. If empty, defaulted to labels on the pod template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", - "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", - "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", - "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", - "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", - "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", - "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", - "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "": "A StatefulSetSpec is the specification of a StatefulSet.", + "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", + "selector": "selector is a label query over pods that should match the replica count. If empty, defaulted to labels on the pod template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", + "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", + "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", + "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", + "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", + "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", + "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", + "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "persistentVolumeClaimRetentionPolicy": "PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.", } func (StatefulSetSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go index cd47542ded14..be3fcc75b62c 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go @@ -500,6 +500,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy { + if in == nil { + return nil + } + out := new(StatefulSetPersistentVolumeClaimRetentionPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in @@ -527,6 +543,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = new(int32) **out = **in } + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = new(StatefulSetPersistentVolumeClaimRetentionPolicy) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go b/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go index e03a95acd470..cd1a06e25f9d 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go +++ b/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go @@ -833,10 +833,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() { var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() { + *m = StatefulSetPersistentVolumeClaimRetentionPolicy{} +} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {} +func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) { + return fileDescriptor_42fe616264472f7e, []int{28} +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src) +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int { + return m.Size() +} +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() { + xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m) +} + +var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo + func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_42fe616264472f7e, []int{28} + return fileDescriptor_42fe616264472f7e, []int{29} } func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -864,7 +894,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_42fe616264472f7e, []int{29} + return fileDescriptor_42fe616264472f7e, []int{30} } func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -892,7 +922,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_42fe616264472f7e, []int{30} + return fileDescriptor_42fe616264472f7e, []int{31} } func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -947,6 +977,7 @@ func init() { proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta2.StatefulSet") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta2.StatefulSetCondition") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta2.StatefulSetList") + proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta2.StatefulSetSpec") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta2.StatefulSetStatus") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta2.StatefulSetUpdateStrategy") @@ -957,144 +988,150 @@ func init() { } var fileDescriptor_42fe616264472f7e = []byte{ - // 2182 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x1c, 0xb7, - 0xf9, 0xf6, 0xec, 0x87, 0xb4, 0x4b, 0x59, 0x92, 0x4d, 0xe9, 0x27, 0x6d, 0xec, 0x5f, 0x57, 0xc6, - 0x26, 0x70, 0x94, 0xd8, 0x9a, 0xb5, 0x95, 0x0f, 0x24, 0x76, 0xd1, 0x56, 0x2b, 0xa5, 0xb6, 0x03, - 0x7d, 0x85, 0xb2, 0x0c, 0x34, 0x68, 0x51, 0x53, 0xbb, 0xf4, 0x6a, 0xa2, 0xf9, 0xc2, 0x0c, 0x67, - 0xeb, 0x45, 0x2f, 0xbd, 0x16, 0x28, 0xd0, 0xf4, 0xda, 0x7f, 0xa2, 0xb7, 0xa2, 0x68, 0x6e, 0x45, - 0x50, 0xf8, 0x18, 0xf4, 0x92, 0x9c, 0x84, 0x7a, 0x73, 0x2a, 0x8a, 0x1e, 0x7b, 0x09, 0x50, 0xa0, - 0x20, 0x87, 0xf3, 0xc1, 0xf9, 0xf0, 0x8e, 0x14, 0x47, 0x69, 0x82, 0xdc, 0xb4, 0xe4, 0xf3, 0x3e, - 0x7c, 0x5f, 0xf2, 0x25, 0xdf, 0x87, 0x1c, 0x81, 0x1f, 0x1d, 0xbd, 0xe5, 0xaa, 0x9a, 0xd5, 0x3e, - 0xf2, 0x0e, 0x88, 0x63, 0x12, 0x4a, 0xdc, 0xf6, 0x80, 0x98, 0x3d, 0xcb, 0x69, 0x8b, 0x0e, 0x6c, - 0x6b, 0x6d, 0x6c, 0xdb, 0x6e, 0x7b, 0x70, 0xf3, 0x80, 0x50, 0xbc, 0xda, 0xee, 0x13, 0x93, 0x38, - 0x98, 0x92, 0x9e, 0x6a, 0x3b, 0x16, 0xb5, 0xe0, 0xa2, 0x0f, 0x54, 0xb1, 0xad, 0xa9, 0x0c, 0xa8, - 0x0a, 0xe0, 0xa5, 0x95, 0xbe, 0x46, 0x0f, 0xbd, 0x03, 0xb5, 0x6b, 0x19, 0xed, 0xbe, 0xd5, 0xb7, - 0xda, 0x1c, 0x7f, 0xe0, 0x3d, 0xe2, 0xbf, 0xf8, 0x0f, 0xfe, 0x97, 0xcf, 0x73, 0xa9, 0x15, 0x1b, - 0xb0, 0x6b, 0x39, 0xa4, 0x3d, 0xb8, 0x99, 0x1c, 0xeb, 0xd2, 0xeb, 0x11, 0xc6, 0xc0, 0xdd, 0x43, - 0xcd, 0x24, 0xce, 0xb0, 0x6d, 0x1f, 0xf5, 0x59, 0x83, 0xdb, 0x36, 0x08, 0xc5, 0x59, 0x56, 0xed, - 0x3c, 0x2b, 0xc7, 0x33, 0xa9, 0x66, 0x90, 0x94, 0xc1, 0x9b, 0xe3, 0x0c, 0xdc, 0xee, 0x21, 0x31, - 0x70, 0xca, 0xee, 0xb5, 0x3c, 0x3b, 0x8f, 0x6a, 0x7a, 0x5b, 0x33, 0xa9, 0x4b, 0x9d, 0xa4, 0x51, - 0xeb, 0xdf, 0x0a, 0x80, 0xeb, 0x96, 0x49, 0x1d, 0x4b, 0xd7, 0x89, 0x83, 0xc8, 0x40, 0x73, 0x35, - 0xcb, 0x84, 0x0f, 0x41, 0x8d, 0xc5, 0xd3, 0xc3, 0x14, 0x37, 0x94, 0x2b, 0xca, 0xf2, 0xd4, 0xea, - 0x0d, 0x35, 0x9a, 0xe9, 0x90, 0x5e, 0xb5, 0x8f, 0xfa, 0xac, 0xc1, 0x55, 0x19, 0x5a, 0x1d, 0xdc, - 0x54, 0x77, 0x0e, 0x3e, 0x20, 0x5d, 0xba, 0x45, 0x28, 0xee, 0xc0, 0x27, 0xc7, 0x4b, 0xe7, 0x46, - 0xc7, 0x4b, 0x20, 0x6a, 0x43, 0x21, 0x2b, 0xdc, 0x01, 0x15, 0xce, 0x5e, 0xe2, 0xec, 0x2b, 0xb9, - 0xec, 0x22, 0x68, 0x15, 0xe1, 0x5f, 0xbc, 0xf3, 0x98, 0x12, 0x93, 0xb9, 0xd7, 0x39, 0x2f, 0xa8, - 0x2b, 0x1b, 0x98, 0x62, 0xc4, 0x89, 0xe0, 0x75, 0x50, 0x73, 0x84, 0xfb, 0x8d, 0xf2, 0x15, 0x65, - 0xb9, 0xdc, 0xb9, 0x20, 0x50, 0xb5, 0x20, 0x2c, 0x14, 0x22, 0x5a, 0x4f, 0x14, 0xb0, 0x90, 0x8e, - 0x7b, 0x53, 0x73, 0x29, 0xfc, 0x69, 0x2a, 0x76, 0xb5, 0x58, 0xec, 0xcc, 0x9a, 0x47, 0x1e, 0x0e, - 0x1c, 0xb4, 0xc4, 0xe2, 0xde, 0x05, 0x55, 0x8d, 0x12, 0xc3, 0x6d, 0x94, 0xae, 0x94, 0x97, 0xa7, - 0x56, 0xaf, 0xa9, 0x39, 0x09, 0xac, 0xa6, 0xbd, 0xeb, 0x4c, 0x0b, 0xde, 0xea, 0x3d, 0xc6, 0x80, - 0x7c, 0xa2, 0xd6, 0xaf, 0x4b, 0xa0, 0xbe, 0x81, 0x89, 0x61, 0x99, 0x7b, 0x84, 0x9e, 0xc1, 0xca, - 0xdd, 0x05, 0x15, 0xd7, 0x26, 0x5d, 0xb1, 0x72, 0x57, 0x73, 0x03, 0x08, 0x7d, 0xda, 0xb3, 0x49, - 0x37, 0x5a, 0x32, 0xf6, 0x0b, 0x71, 0x06, 0xb8, 0x0b, 0x26, 0x5c, 0x8a, 0xa9, 0xe7, 0xf2, 0x05, - 0x9b, 0x5a, 0x5d, 0x2e, 0xc0, 0xc5, 0xf1, 0x9d, 0x19, 0xc1, 0x36, 0xe1, 0xff, 0x46, 0x82, 0xa7, - 0xf5, 0x8f, 0x12, 0x80, 0x21, 0x76, 0xdd, 0x32, 0x7b, 0x1a, 0x65, 0xe9, 0x7c, 0x0b, 0x54, 0xe8, - 0xd0, 0x26, 0x7c, 0x42, 0xea, 0x9d, 0xab, 0x81, 0x2b, 0xf7, 0x87, 0x36, 0xf9, 0xe2, 0x78, 0x69, - 0x21, 0x6d, 0xc1, 0x7a, 0x10, 0xb7, 0x81, 0x9b, 0xa1, 0x93, 0x25, 0x6e, 0xfd, 0xba, 0x3c, 0xf4, - 0x17, 0xc7, 0x4b, 0x19, 0x67, 0x87, 0x1a, 0x32, 0xc9, 0x0e, 0xc2, 0x01, 0x80, 0x3a, 0x76, 0xe9, - 0x7d, 0x07, 0x9b, 0xae, 0x3f, 0x92, 0x66, 0x10, 0x11, 0xfe, 0xab, 0xc5, 0x16, 0x8a, 0x59, 0x74, - 0x2e, 0x09, 0x2f, 0xe0, 0x66, 0x8a, 0x0d, 0x65, 0x8c, 0x00, 0xaf, 0x82, 0x09, 0x87, 0x60, 0xd7, - 0x32, 0x1b, 0x15, 0x1e, 0x45, 0x38, 0x81, 0x88, 0xb7, 0x22, 0xd1, 0x0b, 0x5f, 0x01, 0x93, 0x06, - 0x71, 0x5d, 0xdc, 0x27, 0x8d, 0x2a, 0x07, 0xce, 0x0a, 0xe0, 0xe4, 0x96, 0xdf, 0x8c, 0x82, 0xfe, - 0xd6, 0x1f, 0x15, 0x30, 0x1d, 0xce, 0xdc, 0x19, 0xec, 0x9c, 0x3b, 0xf2, 0xce, 0x69, 0x8d, 0x4f, - 0x96, 0x9c, 0x0d, 0xf3, 0x71, 0x39, 0xe6, 0x38, 0x4b, 0x47, 0xf8, 0x33, 0x50, 0x73, 0x89, 0x4e, - 0xba, 0xd4, 0x72, 0x84, 0xe3, 0xaf, 0x15, 0x74, 0x1c, 0x1f, 0x10, 0x7d, 0x4f, 0x98, 0x76, 0xce, - 0x33, 0xcf, 0x83, 0x5f, 0x28, 0xa4, 0x84, 0xef, 0x81, 0x1a, 0x25, 0x86, 0xad, 0x63, 0x4a, 0xc4, - 0xae, 0x79, 0x31, 0xee, 0x3c, 0xcb, 0x19, 0x46, 0xb6, 0x6b, 0xf5, 0xee, 0x0b, 0x18, 0xdf, 0x32, - 0xe1, 0x64, 0x04, 0xad, 0x28, 0xa4, 0x81, 0x36, 0x98, 0xf1, 0xec, 0x1e, 0x43, 0x52, 0x76, 0x9c, - 0xf7, 0x87, 0x22, 0x87, 0x6e, 0x8c, 0x9f, 0x95, 0x7d, 0xc9, 0xae, 0xb3, 0x20, 0x46, 0x99, 0x91, - 0xdb, 0x51, 0x82, 0x1f, 0xae, 0x81, 0x59, 0x43, 0x33, 0x11, 0xc1, 0xbd, 0xe1, 0x1e, 0xe9, 0x5a, - 0x66, 0xcf, 0xe5, 0xa9, 0x54, 0xed, 0x2c, 0x0a, 0x82, 0xd9, 0x2d, 0xb9, 0x1b, 0x25, 0xf1, 0x70, - 0x13, 0xcc, 0x07, 0x07, 0xf0, 0x5d, 0xcd, 0xa5, 0x96, 0x33, 0xdc, 0xd4, 0x0c, 0x8d, 0x36, 0x26, - 0x38, 0x4f, 0x63, 0x74, 0xbc, 0x34, 0x8f, 0x32, 0xfa, 0x51, 0xa6, 0x55, 0xeb, 0x77, 0x13, 0x60, - 0x36, 0x71, 0x2e, 0xc0, 0x07, 0x60, 0xa1, 0xeb, 0x39, 0x0e, 0x31, 0xe9, 0xb6, 0x67, 0x1c, 0x10, - 0x67, 0xaf, 0x7b, 0x48, 0x7a, 0x9e, 0x4e, 0x7a, 0x7c, 0x59, 0xab, 0x9d, 0xa6, 0xf0, 0x75, 0x61, - 0x3d, 0x13, 0x85, 0x72, 0xac, 0xe1, 0xbb, 0x00, 0x9a, 0xbc, 0x69, 0x4b, 0x73, 0xdd, 0x90, 0xb3, - 0xc4, 0x39, 0xc3, 0xad, 0xb8, 0x9d, 0x42, 0xa0, 0x0c, 0x2b, 0xe6, 0x63, 0x8f, 0xb8, 0x9a, 0x43, - 0x7a, 0x49, 0x1f, 0xcb, 0xb2, 0x8f, 0x1b, 0x99, 0x28, 0x94, 0x63, 0x0d, 0xdf, 0x00, 0x53, 0xfe, - 0x68, 0x7c, 0xce, 0xc5, 0xe2, 0xcc, 0x09, 0xb2, 0xa9, 0xed, 0xa8, 0x0b, 0xc5, 0x71, 0x2c, 0x34, - 0xeb, 0xc0, 0x25, 0xce, 0x80, 0xf4, 0xee, 0xf8, 0xe2, 0x80, 0x55, 0xd0, 0x2a, 0xaf, 0xa0, 0x61, - 0x68, 0x3b, 0x29, 0x04, 0xca, 0xb0, 0x62, 0xa1, 0xf9, 0x59, 0x93, 0x0a, 0x6d, 0x42, 0x0e, 0x6d, - 0x3f, 0x13, 0x85, 0x72, 0xac, 0x59, 0xee, 0xf9, 0x2e, 0xaf, 0x0d, 0xb0, 0xa6, 0xe3, 0x03, 0x9d, - 0x34, 0x26, 0xe5, 0xdc, 0xdb, 0x96, 0xbb, 0x51, 0x12, 0x0f, 0xef, 0x80, 0x8b, 0x7e, 0xd3, 0xbe, - 0x89, 0x43, 0x92, 0x1a, 0x27, 0x79, 0x41, 0x90, 0x5c, 0xdc, 0x4e, 0x02, 0x50, 0xda, 0x06, 0xde, - 0x02, 0x33, 0x5d, 0x4b, 0xd7, 0x79, 0x3e, 0xae, 0x5b, 0x9e, 0x49, 0x1b, 0x75, 0xce, 0x02, 0xd9, - 0x1e, 0x5a, 0x97, 0x7a, 0x50, 0x02, 0x09, 0x7f, 0x0e, 0x40, 0x37, 0x28, 0x0c, 0x6e, 0x03, 0x8c, - 0x51, 0x00, 0xe9, 0xb2, 0x14, 0x55, 0xe6, 0xb0, 0xc9, 0x45, 0x31, 0xca, 0xd6, 0xc7, 0x0a, 0x58, - 0xcc, 0xd9, 0xe8, 0xf0, 0x87, 0x52, 0x11, 0xbc, 0x96, 0x28, 0x82, 0x97, 0x73, 0xcc, 0x62, 0x95, - 0xf0, 0x10, 0x4c, 0x33, 0x41, 0xa2, 0x99, 0x7d, 0x1f, 0x22, 0xce, 0xb2, 0x76, 0x6e, 0x00, 0x28, - 0x8e, 0x8e, 0x4e, 0xe5, 0x8b, 0xa3, 0xe3, 0xa5, 0x69, 0xa9, 0x0f, 0xc9, 0xc4, 0xad, 0xdf, 0x94, - 0x00, 0xd8, 0x20, 0xb6, 0x6e, 0x0d, 0x0d, 0x62, 0x9e, 0x85, 0xa6, 0xb9, 0x27, 0x69, 0x9a, 0x97, - 0xf3, 0x97, 0x24, 0x74, 0x2a, 0x57, 0xd4, 0xbc, 0x97, 0x10, 0x35, 0xaf, 0x14, 0x21, 0x7b, 0xb6, - 0xaa, 0xf9, 0xb4, 0x0c, 0xe6, 0x22, 0x70, 0x24, 0x6b, 0x6e, 0x4b, 0x2b, 0xfa, 0x72, 0x62, 0x45, - 0x17, 0x33, 0x4c, 0xbe, 0x32, 0x5d, 0xf3, 0x01, 0x98, 0x61, 0xaa, 0xc3, 0x5f, 0x3f, 0xae, 0x69, - 0x26, 0x4e, 0xac, 0x69, 0xc2, 0x4a, 0xb4, 0x29, 0x31, 0xa1, 0x04, 0x73, 0x8e, 0x86, 0x9a, 0xfc, - 0x26, 0x6a, 0xa8, 0x3f, 0x29, 0x60, 0x26, 0x5a, 0xa6, 0x33, 0x10, 0x51, 0x77, 0x65, 0x11, 0xf5, - 0x62, 0x81, 0xe4, 0xcc, 0x51, 0x51, 0x9f, 0x56, 0xe2, 0xae, 0x73, 0x19, 0xb5, 0xcc, 0xae, 0x60, - 0xb6, 0xae, 0x75, 0xb1, 0x2b, 0xea, 0xed, 0x79, 0xff, 0xfa, 0xe5, 0xb7, 0xa1, 0xb0, 0x57, 0x12, - 0x5c, 0xa5, 0xaf, 0x56, 0x70, 0x95, 0x9f, 0x8f, 0xe0, 0xfa, 0x09, 0xa8, 0xb9, 0x81, 0xd4, 0xaa, - 0x70, 0xca, 0x6b, 0x85, 0x36, 0xb6, 0x50, 0x59, 0x21, 0x75, 0xa8, 0xaf, 0x42, 0xba, 0x2c, 0x65, - 0x55, 0xfd, 0x3a, 0x95, 0x15, 0x4b, 0x74, 0x1b, 0x7b, 0x2e, 0xe9, 0xf1, 0x4d, 0x55, 0x8b, 0x12, - 0x7d, 0x97, 0xb7, 0x22, 0xd1, 0x0b, 0xf7, 0xc1, 0xa2, 0xed, 0x58, 0x7d, 0x87, 0xb8, 0xee, 0x06, - 0xc1, 0x3d, 0x5d, 0x33, 0x49, 0x10, 0x80, 0x5f, 0x13, 0x2f, 0x8f, 0x8e, 0x97, 0x16, 0x77, 0xb3, - 0x21, 0x28, 0xcf, 0xb6, 0xf5, 0x97, 0x0a, 0xb8, 0x90, 0x3c, 0x1b, 0x73, 0x64, 0x8a, 0x72, 0x2a, - 0x99, 0x72, 0x3d, 0x96, 0xa7, 0xbe, 0x86, 0x8b, 0x3d, 0x15, 0xa4, 0x72, 0x75, 0x0d, 0xcc, 0x0a, - 0x59, 0x12, 0x74, 0x0a, 0xa1, 0x16, 0x2e, 0xcf, 0xbe, 0xdc, 0x8d, 0x92, 0x78, 0x78, 0x1b, 0x4c, - 0x3b, 0x5c, 0x79, 0x05, 0x04, 0xbe, 0x7a, 0xf9, 0x3f, 0x41, 0x30, 0x8d, 0xe2, 0x9d, 0x48, 0xc6, - 0x32, 0xe5, 0x12, 0x09, 0x92, 0x80, 0xa0, 0x22, 0x2b, 0x97, 0xb5, 0x24, 0x00, 0xa5, 0x6d, 0xe0, - 0x16, 0x98, 0xf3, 0xcc, 0x34, 0x95, 0x9f, 0x6b, 0x97, 0x05, 0xd5, 0xdc, 0x7e, 0x1a, 0x82, 0xb2, - 0xec, 0xe0, 0x43, 0x49, 0xcc, 0x4c, 0xf0, 0xf3, 0xe4, 0x7a, 0x81, 0x3d, 0x51, 0x58, 0xcd, 0x64, - 0x48, 0xad, 0x5a, 0x51, 0xa9, 0xd5, 0xfa, 0x48, 0x01, 0x30, 0xbd, 0x0f, 0xc7, 0xbe, 0x04, 0xa4, - 0x2c, 0x62, 0x15, 0x53, 0xcb, 0xd6, 0x3f, 0x37, 0x0a, 0xea, 0x9f, 0xe8, 0x40, 0x2d, 0x26, 0x80, - 0xc4, 0x44, 0x9f, 0xcd, 0xa3, 0x4e, 0x51, 0x01, 0x14, 0x39, 0xf5, 0x1c, 0x04, 0x50, 0x8c, 0xec, - 0xd9, 0x02, 0xe8, 0x9f, 0x25, 0x30, 0x17, 0x81, 0x0b, 0x0b, 0xa0, 0x0c, 0x93, 0xef, 0x1e, 0x76, - 0x8a, 0x89, 0x92, 0x68, 0xea, 0xfe, 0x97, 0x44, 0x49, 0xe4, 0x55, 0x8e, 0x28, 0xf9, 0x43, 0x29, - 0xee, 0xfa, 0x09, 0x45, 0xc9, 0x73, 0x78, 0xe1, 0xf8, 0xc6, 0xe9, 0x9a, 0xd6, 0x5f, 0xcb, 0xe0, - 0x42, 0x72, 0x1f, 0x4a, 0x05, 0x52, 0x19, 0x5b, 0x20, 0x77, 0xc1, 0xfc, 0x23, 0x4f, 0xd7, 0x87, - 0x3c, 0x86, 0x58, 0x95, 0xf4, 0x4b, 0xeb, 0xff, 0x0b, 0xcb, 0xf9, 0x1f, 0x67, 0x60, 0x50, 0xa6, - 0x65, 0xba, 0x5e, 0x56, 0xbe, 0x6c, 0xbd, 0xac, 0x9e, 0xa2, 0x5e, 0x66, 0x4b, 0x8e, 0xf2, 0xa9, - 0x24, 0xc7, 0xc9, 0x8a, 0x65, 0xc6, 0xc1, 0x35, 0xf6, 0xea, 0x3f, 0x52, 0xc0, 0x42, 0xf6, 0x85, - 0x1b, 0xea, 0x60, 0xc6, 0xc0, 0x8f, 0xe3, 0x0f, 0x1f, 0xe3, 0x8a, 0x88, 0x47, 0x35, 0x5d, 0xf5, - 0x3f, 0x19, 0xa9, 0xf7, 0x4c, 0xba, 0xe3, 0xec, 0x51, 0x47, 0x33, 0xfb, 0x7e, 0xe5, 0xdd, 0x92, - 0xb8, 0x50, 0x82, 0x1b, 0xbe, 0x0f, 0x6a, 0x06, 0x7e, 0xbc, 0xe7, 0x39, 0xfd, 0xac, 0x0a, 0x59, - 0x6c, 0x1c, 0xbe, 0x01, 0xb6, 0x04, 0x0b, 0x0a, 0xf9, 0x5a, 0x9f, 0x2b, 0x60, 0x31, 0xa7, 0xaa, - 0x7e, 0x8b, 0xa2, 0xdc, 0x01, 0x57, 0xa4, 0x20, 0xd9, 0xae, 0x24, 0x8f, 0x3c, 0x9d, 0x6f, 0x50, - 0x21, 0x64, 0xae, 0x81, 0xba, 0x8d, 0x1d, 0xaa, 0x85, 0x32, 0xb8, 0xda, 0x99, 0x1e, 0x1d, 0x2f, - 0xd5, 0x77, 0x83, 0x46, 0x14, 0xf5, 0xb7, 0xfe, 0xa3, 0x80, 0xea, 0x5e, 0x17, 0xeb, 0xe4, 0x0c, - 0x94, 0xc4, 0x86, 0xa4, 0x24, 0xf2, 0x5f, 0xe9, 0xb9, 0x3f, 0xb9, 0x22, 0x62, 0x33, 0x21, 0x22, - 0x5e, 0x1a, 0xc3, 0xf3, 0x6c, 0xfd, 0xf0, 0x36, 0xa8, 0x87, 0xc3, 0x9d, 0xec, 0x70, 0x6b, 0xfd, - 0xbe, 0x04, 0xa6, 0x62, 0x43, 0x9c, 0xf0, 0x68, 0x7c, 0x28, 0xd5, 0x03, 0xb6, 0xe9, 0x57, 0x8b, - 0x04, 0xa2, 0x06, 0x67, 0xff, 0x3b, 0x26, 0x75, 0xe2, 0x97, 0xc7, 0x74, 0x49, 0xf8, 0x01, 0x98, - 0xa1, 0xd8, 0xe9, 0x13, 0x1a, 0xf4, 0xf1, 0x09, 0xab, 0x47, 0x8f, 0x29, 0xf7, 0xa5, 0x5e, 0x94, - 0x40, 0x5f, 0xba, 0x0d, 0xa6, 0xa5, 0xc1, 0xe0, 0x05, 0x50, 0x3e, 0x22, 0x43, 0x5f, 0x52, 0x21, - 0xf6, 0x27, 0x9c, 0x07, 0xd5, 0x01, 0xd6, 0x3d, 0x3f, 0xcf, 0xeb, 0xc8, 0xff, 0x71, 0xab, 0xf4, - 0x96, 0xd2, 0xfa, 0x2d, 0x9b, 0x9c, 0x28, 0x39, 0xcf, 0x20, 0xbb, 0xde, 0x95, 0xb2, 0x2b, 0xff, - 0x83, 0x61, 0x7c, 0xcb, 0xe4, 0xe5, 0x18, 0x4a, 0xe4, 0xd8, 0xab, 0x85, 0xd8, 0x9e, 0x9d, 0x69, - 0xff, 0x2a, 0x81, 0xf9, 0x18, 0x3a, 0x92, 0xaa, 0xdf, 0x97, 0xa4, 0xea, 0x72, 0x42, 0xaa, 0x36, - 0xb2, 0x6c, 0xbe, 0xd3, 0xaa, 0xe3, 0xb5, 0xea, 0x9f, 0x15, 0x30, 0x1b, 0x9b, 0xbb, 0x33, 0x10, - 0xab, 0xf7, 0x64, 0xb1, 0xfa, 0x52, 0x91, 0xa4, 0xc9, 0x51, 0xab, 0x1f, 0x4e, 0x48, 0xce, 0x7f, - 0xeb, 0xdf, 0xd0, 0x7e, 0x09, 0xe6, 0x07, 0x96, 0xee, 0x19, 0x64, 0x5d, 0xc7, 0x9a, 0x11, 0x00, - 0x98, 0xba, 0x2b, 0x27, 0xef, 0x89, 0x21, 0x3d, 0x71, 0x5c, 0xcd, 0xa5, 0xc4, 0xa4, 0x0f, 0x22, - 0xcb, 0x48, 0x53, 0x3e, 0xc8, 0xa0, 0x43, 0x99, 0x83, 0xc0, 0x37, 0xc0, 0x14, 0x53, 0x65, 0x5a, - 0x97, 0x6c, 0x63, 0x23, 0x48, 0xac, 0xf0, 0xf3, 0xd8, 0x5e, 0xd4, 0x85, 0xe2, 0x38, 0x78, 0x08, - 0xe6, 0x6c, 0xab, 0xb7, 0x85, 0x4d, 0xdc, 0x27, 0x4c, 0x66, 0xec, 0x5a, 0xba, 0xd6, 0x1d, 0xf2, - 0x87, 0xb5, 0x7a, 0xe7, 0xcd, 0xe0, 0xd1, 0x64, 0x37, 0x0d, 0x61, 0x17, 0xd0, 0x8c, 0x66, 0xbe, - 0xa9, 0xb3, 0x28, 0xa1, 0x93, 0xfa, 0xa4, 0xeb, 0x3f, 0x69, 0xaf, 0x16, 0xc9, 0xb0, 0x53, 0x7e, - 0xd4, 0xcd, 0x7b, 0x37, 0xac, 0x9d, 0xea, 0xdd, 0x30, 0xe3, 0x02, 0x55, 0x3f, 0xd9, 0x05, 0xaa, - 0xf5, 0x51, 0x15, 0x5c, 0x4c, 0x9d, 0xb6, 0x5f, 0xe3, 0xe3, 0x5f, 0xea, 0x26, 0x52, 0x3e, 0xc1, - 0x4d, 0x64, 0x0d, 0xcc, 0x8a, 0xef, 0xc9, 0x89, 0x8b, 0x4c, 0x38, 0x1f, 0xeb, 0x72, 0x37, 0x4a, - 0xe2, 0xb3, 0x1e, 0x1f, 0xab, 0x27, 0x7c, 0x7c, 0x8c, 0x7b, 0x21, 0xfe, 0x3f, 0xca, 0xcf, 0xde, - 0xb4, 0x17, 0xe2, 0xdf, 0xa4, 0x92, 0x78, 0x26, 0x32, 0x7c, 0xd6, 0x90, 0x61, 0x52, 0x16, 0x19, - 0xfb, 0x52, 0x2f, 0x4a, 0xa0, 0xbf, 0xd4, 0x37, 0x53, 0x9c, 0xf1, 0xcd, 0x74, 0xa5, 0xc8, 0x96, - 0x28, 0xfe, 0xce, 0x98, 0x79, 0x63, 0x9c, 0x3a, 0xf9, 0x8d, 0xb1, 0xf5, 0x37, 0x05, 0xbc, 0x90, - 0xbb, 0x29, 0xe1, 0x9a, 0x24, 0x01, 0x56, 0x12, 0x12, 0xe0, 0x7b, 0xb9, 0x86, 0x31, 0x1d, 0xe0, - 0x64, 0x3f, 0x41, 0xbe, 0x5d, 0xec, 0x09, 0x32, 0xe3, 0x1e, 0x31, 0xfe, 0x2d, 0xb2, 0xb3, 0xf2, - 0xe4, 0x69, 0xf3, 0xdc, 0x27, 0x4f, 0x9b, 0xe7, 0x3e, 0x7b, 0xda, 0x3c, 0xf7, 0xab, 0x51, 0x53, - 0x79, 0x32, 0x6a, 0x2a, 0x9f, 0x8c, 0x9a, 0xca, 0x67, 0xa3, 0xa6, 0xf2, 0xf7, 0x51, 0x53, 0xf9, - 0xf0, 0xf3, 0xe6, 0xb9, 0xf7, 0x27, 0xc5, 0x88, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xda, 0x5d, - 0xee, 0xc9, 0xd4, 0x29, 0x00, 0x00, + // 2284 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcf, 0x6f, 0x1b, 0xc7, + 0xf5, 0xf7, 0xf2, 0x87, 0x44, 0x8e, 0x2c, 0xc9, 0x1e, 0xe9, 0x2b, 0x31, 0xf2, 0xb7, 0xa4, 0xb1, + 0x31, 0x1c, 0x25, 0xb6, 0x48, 0x5b, 0xf9, 0x81, 0xc4, 0x2e, 0x92, 0x8a, 0x52, 0x6a, 0x3b, 0xd0, + 0x0f, 0x66, 0x64, 0x39, 0x68, 0xd0, 0x1f, 0x1e, 0x91, 0x63, 0x6a, 0xa3, 0xe5, 0xee, 0x62, 0x77, + 0x96, 0x31, 0xd1, 0x4b, 0xaf, 0x05, 0x0a, 0xb4, 0xbd, 0xf6, 0x9f, 0xe8, 0xad, 0x28, 0x9a, 0x5b, + 0x11, 0x04, 0x3e, 0x06, 0xbd, 0x24, 0xbd, 0x10, 0x35, 0x73, 0x2a, 0x8a, 0xde, 0xda, 0x8b, 0x81, + 0x02, 0xc5, 0xcc, 0xce, 0xfe, 0xde, 0x35, 0x97, 0x8a, 0xa3, 0x34, 0x41, 0x6e, 0xe2, 0xbc, 0xf7, + 0x3e, 0xf3, 0xde, 0xcc, 0x7b, 0xf3, 0x3e, 0x33, 0x2b, 0xf0, 0x83, 0xe3, 0xd7, 0xad, 0xba, 0xa2, + 0x37, 0x8e, 0xed, 0x43, 0x62, 0x6a, 0x84, 0x12, 0xab, 0xd1, 0x27, 0x5a, 0x47, 0x37, 0x1b, 0x42, + 0x80, 0x0d, 0xa5, 0x81, 0x0d, 0xc3, 0x6a, 0xf4, 0xaf, 0x1f, 0x12, 0x8a, 0xd7, 0x1b, 0x5d, 0xa2, + 0x11, 0x13, 0x53, 0xd2, 0xa9, 0x1b, 0xa6, 0x4e, 0x75, 0xb8, 0xec, 0x28, 0xd6, 0xb1, 0xa1, 0xd4, + 0x99, 0x62, 0x5d, 0x28, 0xae, 0xac, 0x75, 0x15, 0x7a, 0x64, 0x1f, 0xd6, 0xdb, 0x7a, 0xaf, 0xd1, + 0xd5, 0xbb, 0x7a, 0x83, 0xeb, 0x1f, 0xda, 0x0f, 0xf8, 0x2f, 0xfe, 0x83, 0xff, 0xe5, 0xe0, 0xac, + 0xc8, 0x81, 0x09, 0xdb, 0xba, 0x49, 0x1a, 0xfd, 0xeb, 0xd1, 0xb9, 0x56, 0x5e, 0xf1, 0x75, 0x7a, + 0xb8, 0x7d, 0xa4, 0x68, 0xc4, 0x1c, 0x34, 0x8c, 0xe3, 0x2e, 0x1b, 0xb0, 0x1a, 0x3d, 0x42, 0x71, + 0x92, 0x55, 0x23, 0xcd, 0xca, 0xb4, 0x35, 0xaa, 0xf4, 0x48, 0xcc, 0xe0, 0xb5, 0x71, 0x06, 0x56, + 0xfb, 0x88, 0xf4, 0x70, 0xcc, 0xee, 0xe5, 0x34, 0x3b, 0x9b, 0x2a, 0x6a, 0x43, 0xd1, 0xa8, 0x45, + 0xcd, 0xa8, 0x91, 0xfc, 0x6f, 0x09, 0xc0, 0x4d, 0x5d, 0xa3, 0xa6, 0xae, 0xaa, 0xc4, 0x44, 0xa4, + 0xaf, 0x58, 0x8a, 0xae, 0xc1, 0xfb, 0xa0, 0xc4, 0xe2, 0xe9, 0x60, 0x8a, 0x2b, 0xd2, 0x45, 0x69, + 0x75, 0x66, 0xfd, 0x5a, 0xdd, 0x5f, 0x69, 0x0f, 0xbe, 0x6e, 0x1c, 0x77, 0xd9, 0x80, 0x55, 0x67, + 0xda, 0xf5, 0xfe, 0xf5, 0xfa, 0xde, 0xe1, 0x07, 0xa4, 0x4d, 0x77, 0x08, 0xc5, 0x4d, 0xf8, 0x68, + 0x58, 0x3b, 0x33, 0x1a, 0xd6, 0x80, 0x3f, 0x86, 0x3c, 0x54, 0xb8, 0x07, 0x0a, 0x1c, 0x3d, 0xc7, + 0xd1, 0xd7, 0x52, 0xd1, 0x45, 0xd0, 0x75, 0x84, 0x3f, 0x7c, 0xfb, 0x21, 0x25, 0x1a, 0x73, 0xaf, + 0x79, 0x56, 0x40, 0x17, 0xb6, 0x30, 0xc5, 0x88, 0x03, 0xc1, 0xab, 0xa0, 0x64, 0x0a, 0xf7, 0x2b, + 0xf9, 0x8b, 0xd2, 0x6a, 0xbe, 0x79, 0x4e, 0x68, 0x95, 0xdc, 0xb0, 0x90, 0xa7, 0x21, 0x3f, 0x92, + 0xc0, 0x52, 0x3c, 0xee, 0x6d, 0xc5, 0xa2, 0xf0, 0xc7, 0xb1, 0xd8, 0xeb, 0xd9, 0x62, 0x67, 0xd6, + 0x3c, 0x72, 0x6f, 0x62, 0x77, 0x24, 0x10, 0x77, 0x0b, 0x14, 0x15, 0x4a, 0x7a, 0x56, 0x25, 0x77, + 0x31, 0xbf, 0x3a, 0xb3, 0x7e, 0xa5, 0x9e, 0x92, 0xc0, 0xf5, 0xb8, 0x77, 0xcd, 0x59, 0x81, 0x5b, + 0xbc, 0xc3, 0x10, 0x90, 0x03, 0x24, 0xff, 0x32, 0x07, 0xca, 0x5b, 0x98, 0xf4, 0x74, 0x6d, 0x9f, + 0xd0, 0x53, 0xd8, 0xb9, 0xdb, 0xa0, 0x60, 0x19, 0xa4, 0x2d, 0x76, 0xee, 0x72, 0x6a, 0x00, 0x9e, + 0x4f, 0xfb, 0x06, 0x69, 0xfb, 0x5b, 0xc6, 0x7e, 0x21, 0x8e, 0x00, 0x5b, 0x60, 0xca, 0xa2, 0x98, + 0xda, 0x16, 0xdf, 0xb0, 0x99, 0xf5, 0xd5, 0x0c, 0x58, 0x5c, 0xbf, 0x39, 0x27, 0xd0, 0xa6, 0x9c, + 0xdf, 0x48, 0xe0, 0xc8, 0x7f, 0xcf, 0x01, 0xe8, 0xe9, 0x6e, 0xea, 0x5a, 0x47, 0xa1, 0x2c, 0x9d, + 0x6f, 0x80, 0x02, 0x1d, 0x18, 0x84, 0x2f, 0x48, 0xb9, 0x79, 0xd9, 0x75, 0xe5, 0xee, 0xc0, 0x20, + 0x4f, 0x86, 0xb5, 0xa5, 0xb8, 0x05, 0x93, 0x20, 0x6e, 0x03, 0xb7, 0x3d, 0x27, 0x73, 0xdc, 0xfa, + 0x95, 0xf0, 0xd4, 0x4f, 0x86, 0xb5, 0x84, 0xb3, 0xa3, 0xee, 0x21, 0x85, 0x1d, 0x84, 0x7d, 0x00, + 0x55, 0x6c, 0xd1, 0xbb, 0x26, 0xd6, 0x2c, 0x67, 0x26, 0xa5, 0x47, 0x44, 0xf8, 0x2f, 0x65, 0xdb, + 0x28, 0x66, 0xd1, 0x5c, 0x11, 0x5e, 0xc0, 0xed, 0x18, 0x1a, 0x4a, 0x98, 0x01, 0x5e, 0x06, 0x53, + 0x26, 0xc1, 0x96, 0xae, 0x55, 0x0a, 0x3c, 0x0a, 0x6f, 0x01, 0x11, 0x1f, 0x45, 0x42, 0x0a, 0x5f, + 0x04, 0xd3, 0x3d, 0x62, 0x59, 0xb8, 0x4b, 0x2a, 0x45, 0xae, 0x38, 0x2f, 0x14, 0xa7, 0x77, 0x9c, + 0x61, 0xe4, 0xca, 0xe5, 0x3f, 0x48, 0x60, 0xd6, 0x5b, 0xb9, 0x53, 0xa8, 0x9c, 0x5b, 0xe1, 0xca, + 0x91, 0xc7, 0x27, 0x4b, 0x4a, 0xc1, 0x7c, 0x9c, 0x0f, 0x38, 0xce, 0xd2, 0x11, 0xfe, 0x04, 0x94, + 0x2c, 0xa2, 0x92, 0x36, 0xd5, 0x4d, 0xe1, 0xf8, 0xcb, 0x19, 0x1d, 0xc7, 0x87, 0x44, 0xdd, 0x17, + 0xa6, 0xcd, 0xb3, 0xcc, 0x73, 0xf7, 0x17, 0xf2, 0x20, 0xe1, 0xbb, 0xa0, 0x44, 0x49, 0xcf, 0x50, + 0x31, 0x25, 0xa2, 0x6a, 0x9e, 0x0f, 0x3a, 0xcf, 0x72, 0x86, 0x81, 0xb5, 0xf4, 0xce, 0x5d, 0xa1, + 0xc6, 0x4b, 0xc6, 0x5b, 0x0c, 0x77, 0x14, 0x79, 0x30, 0xd0, 0x00, 0x73, 0xb6, 0xd1, 0x61, 0x9a, + 0x94, 0x1d, 0xe7, 0xdd, 0x81, 0xc8, 0xa1, 0x6b, 0xe3, 0x57, 0xe5, 0x20, 0x64, 0xd7, 0x5c, 0x12, + 0xb3, 0xcc, 0x85, 0xc7, 0x51, 0x04, 0x1f, 0x6e, 0x80, 0xf9, 0x9e, 0xa2, 0x21, 0x82, 0x3b, 0x83, + 0x7d, 0xd2, 0xd6, 0xb5, 0x8e, 0xc5, 0x53, 0xa9, 0xd8, 0x5c, 0x16, 0x00, 0xf3, 0x3b, 0x61, 0x31, + 0x8a, 0xea, 0xc3, 0x6d, 0xb0, 0xe8, 0x1e, 0xc0, 0xb7, 0x15, 0x8b, 0xea, 0xe6, 0x60, 0x5b, 0xe9, + 0x29, 0xb4, 0x32, 0xc5, 0x71, 0x2a, 0xa3, 0x61, 0x6d, 0x11, 0x25, 0xc8, 0x51, 0xa2, 0x95, 0xfc, + 0xdb, 0x29, 0x30, 0x1f, 0x39, 0x17, 0xe0, 0x3d, 0xb0, 0xd4, 0xb6, 0x4d, 0x93, 0x68, 0x74, 0xd7, + 0xee, 0x1d, 0x12, 0x73, 0xbf, 0x7d, 0x44, 0x3a, 0xb6, 0x4a, 0x3a, 0x7c, 0x5b, 0x8b, 0xcd, 0xaa, + 0xf0, 0x75, 0x69, 0x33, 0x51, 0x0b, 0xa5, 0x58, 0xc3, 0x77, 0x00, 0xd4, 0xf8, 0xd0, 0x8e, 0x62, + 0x59, 0x1e, 0x66, 0x8e, 0x63, 0x7a, 0xa5, 0xb8, 0x1b, 0xd3, 0x40, 0x09, 0x56, 0xcc, 0xc7, 0x0e, + 0xb1, 0x14, 0x93, 0x74, 0xa2, 0x3e, 0xe6, 0xc3, 0x3e, 0x6e, 0x25, 0x6a, 0xa1, 0x14, 0x6b, 0xf8, + 0x2a, 0x98, 0x71, 0x66, 0xe3, 0x6b, 0x2e, 0x36, 0x67, 0x41, 0x80, 0xcd, 0xec, 0xfa, 0x22, 0x14, + 0xd4, 0x63, 0xa1, 0xe9, 0x87, 0x16, 0x31, 0xfb, 0xa4, 0x73, 0xcb, 0x21, 0x07, 0xac, 0x83, 0x16, + 0x79, 0x07, 0xf5, 0x42, 0xdb, 0x8b, 0x69, 0xa0, 0x04, 0x2b, 0x16, 0x9a, 0x93, 0x35, 0xb1, 0xd0, + 0xa6, 0xc2, 0xa1, 0x1d, 0x24, 0x6a, 0xa1, 0x14, 0x6b, 0x96, 0x7b, 0x8e, 0xcb, 0x1b, 0x7d, 0xac, + 0xa8, 0xf8, 0x50, 0x25, 0x95, 0xe9, 0x70, 0xee, 0xed, 0x86, 0xc5, 0x28, 0xaa, 0x0f, 0x6f, 0x81, + 0xf3, 0xce, 0xd0, 0x81, 0x86, 0x3d, 0x90, 0x12, 0x07, 0x79, 0x4e, 0x80, 0x9c, 0xdf, 0x8d, 0x2a, + 0xa0, 0xb8, 0x0d, 0xbc, 0x01, 0xe6, 0xda, 0xba, 0xaa, 0xf2, 0x7c, 0xdc, 0xd4, 0x6d, 0x8d, 0x56, + 0xca, 0x1c, 0x05, 0xb2, 0x1a, 0xda, 0x0c, 0x49, 0x50, 0x44, 0x13, 0xfe, 0x0c, 0x80, 0xb6, 0xdb, + 0x18, 0xac, 0x0a, 0x18, 0xc3, 0x00, 0xe2, 0x6d, 0xc9, 0xef, 0xcc, 0xde, 0x90, 0x85, 0x02, 0x90, + 0xf2, 0xc7, 0x12, 0x58, 0x4e, 0x29, 0x74, 0xf8, 0x56, 0xa8, 0x09, 0x5e, 0x89, 0x34, 0xc1, 0x0b, + 0x29, 0x66, 0x81, 0x4e, 0x78, 0x04, 0x66, 0x19, 0x21, 0x51, 0xb4, 0xae, 0xa3, 0x22, 0xce, 0xb2, + 0x46, 0x6a, 0x00, 0x28, 0xa8, 0xed, 0x9f, 0xca, 0xe7, 0x47, 0xc3, 0xda, 0x6c, 0x48, 0x86, 0xc2, + 0xc0, 0xf2, 0xaf, 0x72, 0x00, 0x6c, 0x11, 0x43, 0xd5, 0x07, 0x3d, 0xa2, 0x9d, 0x06, 0xa7, 0xb9, + 0x13, 0xe2, 0x34, 0x2f, 0xa4, 0x6f, 0x89, 0xe7, 0x54, 0x2a, 0xa9, 0x79, 0x37, 0x42, 0x6a, 0x5e, + 0xcc, 0x02, 0xf6, 0x74, 0x56, 0xf3, 0x59, 0x1e, 0x2c, 0xf8, 0xca, 0x3e, 0xad, 0xb9, 0x19, 0xda, + 0xd1, 0x17, 0x22, 0x3b, 0xba, 0x9c, 0x60, 0xf2, 0x95, 0xf1, 0x9a, 0x0f, 0xc0, 0x1c, 0x63, 0x1d, + 0xce, 0xfe, 0x71, 0x4e, 0x33, 0x35, 0x31, 0xa7, 0xf1, 0x3a, 0xd1, 0x76, 0x08, 0x09, 0x45, 0x90, + 0x53, 0x38, 0xd4, 0xf4, 0x37, 0x91, 0x43, 0xfd, 0x51, 0x02, 0x73, 0xfe, 0x36, 0x9d, 0x02, 0x89, + 0xba, 0x1d, 0x26, 0x51, 0xcf, 0x67, 0x48, 0xce, 0x14, 0x16, 0xf5, 0x59, 0x21, 0xe8, 0x3a, 0xa7, + 0x51, 0xab, 0xec, 0x0a, 0x66, 0xa8, 0x4a, 0x1b, 0x5b, 0xa2, 0xdf, 0x9e, 0x75, 0xae, 0x5f, 0xce, + 0x18, 0xf2, 0xa4, 0x21, 0xc2, 0x95, 0xfb, 0x6a, 0x09, 0x57, 0xfe, 0xd9, 0x10, 0xae, 0x1f, 0x81, + 0x92, 0xe5, 0x52, 0xad, 0x02, 0x87, 0xbc, 0x92, 0xa9, 0xb0, 0x05, 0xcb, 0xf2, 0xa0, 0x3d, 0x7e, + 0xe5, 0xc1, 0x25, 0x31, 0xab, 0xe2, 0xd7, 0xc9, 0xac, 0x58, 0xa2, 0x1b, 0xd8, 0xb6, 0x48, 0x87, + 0x17, 0x55, 0xc9, 0x4f, 0xf4, 0x16, 0x1f, 0x45, 0x42, 0x0a, 0x0f, 0xc0, 0xb2, 0x61, 0xea, 0x5d, + 0x93, 0x58, 0xd6, 0x16, 0xc1, 0x1d, 0x55, 0xd1, 0x88, 0x1b, 0x80, 0xd3, 0x13, 0x2f, 0x8c, 0x86, + 0xb5, 0xe5, 0x56, 0xb2, 0x0a, 0x4a, 0xb3, 0x95, 0xff, 0x5c, 0x00, 0xe7, 0xa2, 0x67, 0x63, 0x0a, + 0x4d, 0x91, 0x4e, 0x44, 0x53, 0xae, 0x06, 0xf2, 0xd4, 0xe1, 0x70, 0x81, 0xa7, 0x82, 0x58, 0xae, + 0x6e, 0x80, 0x79, 0x41, 0x4b, 0x5c, 0xa1, 0x20, 0x6a, 0xde, 0xf6, 0x1c, 0x84, 0xc5, 0x28, 0xaa, + 0x0f, 0x6f, 0x82, 0x59, 0x93, 0x33, 0x2f, 0x17, 0xc0, 0x61, 0x2f, 0xff, 0x27, 0x00, 0x66, 0x51, + 0x50, 0x88, 0xc2, 0xba, 0x8c, 0xb9, 0xf8, 0x84, 0xc4, 0x05, 0x28, 0x84, 0x99, 0xcb, 0x46, 0x54, + 0x01, 0xc5, 0x6d, 0xe0, 0x0e, 0x58, 0xb0, 0xb5, 0x38, 0x94, 0x93, 0x6b, 0x17, 0x04, 0xd4, 0xc2, + 0x41, 0x5c, 0x05, 0x25, 0xd9, 0xc1, 0xfb, 0x21, 0x32, 0x33, 0xc5, 0xcf, 0x93, 0xab, 0x19, 0x6a, + 0x22, 0x33, 0x9b, 0x49, 0xa0, 0x5a, 0xa5, 0xac, 0x54, 0x4b, 0xfe, 0x48, 0x02, 0x30, 0x5e, 0x87, + 0x63, 0x5f, 0x02, 0x62, 0x16, 0x81, 0x8e, 0xa9, 0x24, 0xf3, 0x9f, 0x6b, 0x19, 0xf9, 0x8f, 0x7f, + 0xa0, 0x66, 0x23, 0x40, 0x62, 0xa1, 0x4f, 0xe7, 0x51, 0x27, 0x2b, 0x01, 0xf2, 0x9d, 0x7a, 0x06, + 0x04, 0x28, 0x00, 0xf6, 0x74, 0x02, 0xf4, 0x8f, 0x1c, 0x58, 0xf0, 0x95, 0x33, 0x13, 0xa0, 0x04, + 0x93, 0xef, 0x1e, 0x76, 0xb2, 0x91, 0x12, 0x7f, 0xe9, 0xfe, 0x97, 0x48, 0x89, 0xef, 0x55, 0x0a, + 0x29, 0xf9, 0x7d, 0x2e, 0xe8, 0xfa, 0x84, 0xa4, 0xe4, 0x19, 0xbc, 0x70, 0x7c, 0xe3, 0x78, 0x8d, + 0xfc, 0x49, 0x1e, 0x9c, 0x8b, 0xd6, 0x61, 0xa8, 0x41, 0x4a, 0x63, 0x1b, 0x64, 0x0b, 0x2c, 0x3e, + 0xb0, 0x55, 0x75, 0xc0, 0x63, 0x08, 0x74, 0x49, 0xa7, 0xb5, 0xfe, 0xbf, 0xb0, 0x5c, 0xfc, 0x61, + 0x82, 0x0e, 0x4a, 0xb4, 0x8c, 0xf7, 0xcb, 0xc2, 0x97, 0xed, 0x97, 0xc5, 0x13, 0xf4, 0xcb, 0x64, + 0xca, 0x91, 0x3f, 0x11, 0xe5, 0x98, 0xac, 0x59, 0x26, 0x1c, 0x5c, 0x63, 0xaf, 0xfe, 0x23, 0x09, + 0x2c, 0x25, 0x5f, 0xb8, 0xa1, 0x0a, 0xe6, 0x7a, 0xf8, 0x61, 0xf0, 0xe1, 0x63, 0x5c, 0x13, 0xb1, + 0xa9, 0xa2, 0xd6, 0x9d, 0x4f, 0x46, 0xf5, 0x3b, 0x1a, 0xdd, 0x33, 0xf7, 0xa9, 0xa9, 0x68, 0x5d, + 0xa7, 0xf3, 0xee, 0x84, 0xb0, 0x50, 0x04, 0x1b, 0xbe, 0x0f, 0x4a, 0x3d, 0xfc, 0x70, 0xdf, 0x36, + 0xbb, 0x49, 0x1d, 0x32, 0xdb, 0x3c, 0xbc, 0x00, 0x76, 0x04, 0x0a, 0xf2, 0xf0, 0xe4, 0x2f, 0x24, + 0xb0, 0x9c, 0xd2, 0x55, 0xbf, 0x45, 0x51, 0xee, 0x81, 0x8b, 0xa1, 0x20, 0x59, 0x55, 0x92, 0x07, + 0xb6, 0xca, 0x0b, 0x54, 0x10, 0x99, 0x2b, 0xa0, 0x6c, 0x60, 0x93, 0x2a, 0x1e, 0x0d, 0x2e, 0x36, + 0x67, 0x47, 0xc3, 0x5a, 0xb9, 0xe5, 0x0e, 0x22, 0x5f, 0x2e, 0xff, 0x47, 0x02, 0xc5, 0xfd, 0x36, + 0x56, 0xc9, 0x29, 0x30, 0x89, 0xad, 0x10, 0x93, 0x48, 0x7f, 0xa5, 0xe7, 0xfe, 0xa4, 0x92, 0x88, + 0xed, 0x08, 0x89, 0xb8, 0x34, 0x06, 0xe7, 0xe9, 0xfc, 0xe1, 0x0d, 0x50, 0xf6, 0xa6, 0x9b, 0xec, + 0x70, 0x93, 0x7f, 0x97, 0x03, 0x33, 0x81, 0x29, 0x26, 0x3c, 0x1a, 0xef, 0x87, 0xfa, 0x01, 0x2b, + 0xfa, 0xf5, 0x2c, 0x81, 0xd4, 0xdd, 0xb3, 0xff, 0x6d, 0x8d, 0x9a, 0xc1, 0xcb, 0x63, 0xbc, 0x25, + 0xbc, 0x09, 0xe6, 0x28, 0x36, 0xbb, 0x84, 0xba, 0x32, 0xbe, 0x60, 0x65, 0xff, 0x31, 0xe5, 0x6e, + 0x48, 0x8a, 0x22, 0xda, 0x2b, 0x37, 0xc1, 0x6c, 0x68, 0x32, 0x78, 0x0e, 0xe4, 0x8f, 0xc9, 0xc0, + 0xa1, 0x54, 0x88, 0xfd, 0x09, 0x17, 0x41, 0xb1, 0x8f, 0x55, 0xdb, 0xc9, 0xf3, 0x32, 0x72, 0x7e, + 0xdc, 0xc8, 0xbd, 0x2e, 0xc9, 0xbf, 0x66, 0x8b, 0xe3, 0x27, 0xe7, 0x29, 0x64, 0xd7, 0x3b, 0xa1, + 0xec, 0x4a, 0xff, 0x60, 0x18, 0x2c, 0x99, 0xb4, 0x1c, 0x43, 0x91, 0x1c, 0x7b, 0x29, 0x13, 0xda, + 0xd3, 0x33, 0xed, 0x9f, 0x39, 0xb0, 0x18, 0xd0, 0xf6, 0xa9, 0xea, 0xf7, 0x43, 0x54, 0x75, 0x35, + 0x42, 0x55, 0x2b, 0x49, 0x36, 0xdf, 0x71, 0xd5, 0xf1, 0x5c, 0xf5, 0x4f, 0x12, 0x98, 0x0f, 0xac, + 0xdd, 0x29, 0x90, 0xd5, 0x3b, 0x61, 0xb2, 0x7a, 0x29, 0x4b, 0xd2, 0xa4, 0xb0, 0xd5, 0x7f, 0x49, + 0xa0, 0x11, 0xd0, 0x6a, 0x11, 0xd3, 0x52, 0x2c, 0x4a, 0x34, 0x7a, 0x4f, 0x57, 0xed, 0x1e, 0xd9, + 0x54, 0xb1, 0xd2, 0x43, 0x84, 0x0d, 0x28, 0xba, 0xd6, 0xd2, 0x55, 0xa5, 0x3d, 0x80, 0x18, 0xcc, + 0x7c, 0x78, 0x44, 0xb4, 0x2d, 0xa2, 0x12, 0x2a, 0x3e, 0x6b, 0x95, 0x9b, 0x6f, 0xb9, 0x5f, 0x79, + 0xde, 0xf3, 0x45, 0x4f, 0x86, 0xb5, 0xd5, 0x2c, 0x88, 0x3c, 0xcb, 0x82, 0x98, 0xf0, 0xa7, 0x00, + 0xb0, 0x9f, 0xfc, 0x3c, 0xea, 0x88, 0x84, 0x7b, 0xd3, 0xad, 0xca, 0xf7, 0x3c, 0xc9, 0x44, 0x13, + 0x04, 0x10, 0xe5, 0xbf, 0x4e, 0x87, 0xf6, 0xec, 0x5b, 0xff, 0x74, 0xf8, 0x73, 0xb0, 0xd8, 0xf7, + 0x57, 0xc7, 0x55, 0x60, 0xa4, 0x36, 0x1f, 0xbd, 0x1e, 0x7b, 0xf0, 0x49, 0xeb, 0xea, 0x53, 0xe9, + 0x7b, 0x09, 0x70, 0x28, 0x71, 0x12, 0xf8, 0x2a, 0x98, 0x61, 0x64, 0x54, 0x69, 0x93, 0x5d, 0xdc, + 0x73, 0xeb, 0xc9, 0xfb, 0x2a, 0xb8, 0xef, 0x8b, 0x50, 0x50, 0x0f, 0x1e, 0x81, 0x05, 0x43, 0xef, + 0xec, 0x60, 0x0d, 0x77, 0x09, 0x63, 0x57, 0xce, 0x56, 0xf2, 0xf7, 0xc4, 0x72, 0xf3, 0x35, 0xf7, + 0xad, 0xa8, 0x15, 0x57, 0x61, 0xf7, 0xee, 0x84, 0x61, 0x9e, 0x04, 0x49, 0x90, 0xd0, 0x8c, 0x7d, + 0xc9, 0x76, 0x5e, 0xf2, 0xd7, 0xb3, 0x14, 0xd6, 0x09, 0xbf, 0x65, 0xa7, 0x3d, 0x97, 0x96, 0x4e, + 0xf4, 0x5c, 0x9a, 0x70, 0x6f, 0x2c, 0x4f, 0x78, 0x6f, 0xfc, 0x44, 0x02, 0x97, 0x8c, 0x0c, 0xb5, + 0x54, 0x01, 0x7c, 0x6d, 0x6e, 0x67, 0x59, 0x9b, 0x2c, 0xb5, 0xd9, 0x5c, 0x1d, 0x0d, 0x6b, 0x97, + 0xb2, 0x68, 0xa2, 0x4c, 0xfe, 0xc9, 0x1f, 0x15, 0xc1, 0xf9, 0x58, 0xb7, 0xfc, 0x1a, 0x1f, 0x6f, + 0x63, 0x37, 0xc9, 0xfc, 0x04, 0x37, 0xc9, 0x0d, 0x30, 0x2f, 0xfe, 0x1f, 0x20, 0x72, 0x11, 0xf5, + 0x36, 0x76, 0x33, 0x2c, 0x46, 0x51, 0xfd, 0xa4, 0xc7, 0xe3, 0xe2, 0x84, 0x8f, 0xc7, 0x41, 0x2f, + 0xc4, 0xff, 0xb7, 0x39, 0x65, 0x18, 0xf7, 0x42, 0xfc, 0x9b, 0x5b, 0x54, 0x9f, 0x91, 0x44, 0x07, + 0xd5, 0x43, 0x98, 0x0e, 0x93, 0xc4, 0x83, 0x90, 0x14, 0x45, 0xb4, 0xbf, 0xd4, 0x37, 0x6f, 0x9c, + 0xf0, 0xcd, 0x7b, 0x2d, 0x4b, 0xfe, 0x66, 0x7f, 0x27, 0x4e, 0xbc, 0xf1, 0xcf, 0x4c, 0x7e, 0xe3, + 0x97, 0xff, 0x22, 0x81, 0xe7, 0x52, 0x4f, 0x17, 0xb8, 0x11, 0xa2, 0x70, 0x6b, 0x11, 0x0a, 0xf7, + 0xbd, 0x54, 0xc3, 0x00, 0x8f, 0x33, 0x93, 0x9f, 0x90, 0xdf, 0xc8, 0xf6, 0x84, 0x9c, 0x70, 0x0f, + 0x1c, 0xff, 0x96, 0xdc, 0x5c, 0x7b, 0xf4, 0xb8, 0x7a, 0xe6, 0xd3, 0xc7, 0xd5, 0x33, 0x9f, 0x3f, + 0xae, 0x9e, 0xf9, 0xc5, 0xa8, 0x2a, 0x3d, 0x1a, 0x55, 0xa5, 0x4f, 0x47, 0x55, 0xe9, 0xf3, 0x51, + 0x55, 0xfa, 0xdb, 0xa8, 0x2a, 0xfd, 0xe6, 0x8b, 0xea, 0x99, 0xf7, 0xa7, 0xc5, 0x8c, 0xff, 0x0d, + 0x00, 0x00, 0xff, 0xff, 0x3e, 0x13, 0x3b, 0xc7, 0x94, 0x2b, 0x00, 0x00, } func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { @@ -2523,6 +2560,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i -= len(m.WhenScaled) + copy(dAtA[i:], m.WhenScaled) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled))) + i-- + dAtA[i] = 0x12 + i -= len(m.WhenDeleted) + copy(dAtA[i:], m.WhenDeleted) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2543,6 +2613,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.PersistentVolumeClaimRetentionPolicy != nil { + { + size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i-- dAtA[i] = 0x48 @@ -3232,6 +3314,19 @@ func (m *StatefulSetList) Size() (n int) { return n } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.WhenDeleted) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.WhenScaled) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *StatefulSetSpec) Size() (n int) { if m == nil { return 0 @@ -3263,6 +3358,10 @@ func (m *StatefulSetSpec) Size() (n int) { n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) } n += 1 + sovGenerated(uint64(m.MinReadySeconds)) + if m.PersistentVolumeClaimRetentionPolicy != nil { + l = m.PersistentVolumeClaimRetentionPolicy.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -3720,6 +3819,17 @@ func (this *StatefulSetList) String() string { }, "") return s } +func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`, + `WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`, + `WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`, + `}`, + }, "") + return s +} func (this *StatefulSetSpec) String() string { if this == nil { return "nil" @@ -3739,6 +3849,7 @@ func (this *StatefulSetSpec) String() string { `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, + `PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`, `}`, }, "") return s @@ -8228,6 +8339,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error { } return nil } +func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -8516,6 +8741,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PersistentVolumeClaimRetentionPolicy == nil { + m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{} + } + if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/apps/v1beta2/generated.proto b/staging/src/k8s.io/api/apps/v1beta2/generated.proto index c67502769b30..6d9505865a50 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/generated.proto +++ b/staging/src/k8s.io/api/apps/v1beta2/generated.proto @@ -656,6 +656,23 @@ message StatefulSetList { repeated StatefulSet items = 2; } +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +message StatefulSetPersistentVolumeClaimRetentionPolicy { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + optional string whenDeleted = 1; + + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + optional string whenScaled = 2; +} + // A StatefulSetSpec is the specification of a StatefulSet. message StatefulSetSpec { // replicas is the desired number of replicas of the given Template. @@ -722,6 +739,12 @@ message StatefulSetSpec { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional optional int32 minReadySeconds = 9; + + // PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from + // the StatefulSet VolumeClaimTemplates. This requires the + // StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. + // +optional + optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10; } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1beta2/types.go b/staging/src/k8s.io/api/apps/v1beta2/types.go index d8ad98874cc8..332bc7ed82bf 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/types.go +++ b/staging/src/k8s.io/api/apps/v1beta2/types.go @@ -169,6 +169,40 @@ type RollingUpdateStatefulSetStrategy struct { Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` } +// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine +// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is +// deleted or scaled down. +type PersistentVolumeClaimRetentionPolicyType string + +const ( + // RetainPersistentVolumeClaimRetentionPolicyType is the default + // PersistentVolumeClaimRetentionPolicy and specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will not be deleted. + RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain" + // RetentionPersistentVolumeClaimRetentionPolicyType specifies that + // PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates + // will be deleted in the scenario specified in + // StatefulSetPersistentVolumeClaimRetentionPolicy. + RetentionPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs +// created from the StatefulSet VolumeClaimTemplates. +type StatefulSetPersistentVolumeClaimRetentionPolicy struct { + // WhenDeleted specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is deleted. The default policy + // of `Retain` causes PVCs to not be affected by StatefulSet deletion. The + // `Delete` policy causes those PVCs to be deleted. + WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"` + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"` +} + // A StatefulSetSpec is the specification of a StatefulSet. type StatefulSetSpec struct { // replicas is the desired number of replicas of the given Template. @@ -235,6 +269,12 @@ type StatefulSetSpec struct { // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // +optional MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` + + // PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from + // the StatefulSet VolumeClaimTemplates. This requires the + // StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. + // +optional + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"` } // StatefulSetStatus represents the current state of a StatefulSet. diff --git a/staging/src/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go b/staging/src/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go index 46f450d47234..454c632dc257 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/apps/v1beta2/types_swagger_doc_generated.go @@ -351,17 +351,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string { return map_StatefulSetList } +var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{ + "": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.", + "whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.", + "whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.", +} + +func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string { + return map_StatefulSetPersistentVolumeClaimRetentionPolicy +} + var map_StatefulSetSpec = map[string]string{ - "": "A StatefulSetSpec is the specification of a StatefulSet.", - "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", - "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", - "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", - "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", - "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", - "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", - "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", - "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", - "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "": "A StatefulSetSpec is the specification of a StatefulSet.", + "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", + "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", + "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", + "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", + "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", + "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", + "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", + "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", + "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", + "persistentVolumeClaimRetentionPolicy": "PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.", } func (StatefulSetSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go b/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go index b4e955afaf1b..8293b9886b6c 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go @@ -755,6 +755,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy. +func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy { + if in == nil { + return nil + } + out := new(StatefulSetPersistentVolumeClaimRetentionPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in @@ -782,6 +798,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = new(int32) **out = **in } + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = new(StatefulSetPersistentVolumeClaimRetentionPolicy) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.json b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.json index f75e26c5f455..3411b27bdb61 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.json +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.json @@ -1693,26 +1693,30 @@ } }, "revisionHistoryLimit": -1166275743, - "minReadySeconds": -1056262432 + "minReadySeconds": -1056262432, + "persistentVolumeClaimRetentionPolicy": { + "whenDeleted": "/Ş槀墺=Ĉ鳟/d", + "whenScaled": "á" + } }, "status": { - "observedGeneration": -3770279213092072518, - "replicas": -1669166259, - "readyReplicas": -175399547, - "currentReplicas": 223338274, - "updatedReplicas": -1279136912, + "observedGeneration": 5897193066236579633, + "replicas": 1442750499, + "readyReplicas": -1190434752, + "currentReplicas": -1929177074, + "updatedReplicas": -690843774, "currentRevision": "538", "updateRevision": "539", - "collisionCount": 1222237461, + "collisionCount": 1329525670, "conditions": [ { - "type": "壣V礆á¤拈tY", - "status": "飼蒱鄆\u0026嬜Š\u0026?鳢.ǀŭ瘢颦z", - "lastTransitionTime": "2368-07-30T22:05:09Z", + "type": "\u0026嬜Š\u0026?鳢.ǀŭ瘢", + "status": "Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6", + "lastTransitionTime": "2881-11-22T04:01:01Z", "reason": "540", "message": "541" } ], - "availableReplicas": -1174483980 + "availableReplicas": -1889018254 } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.pb b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.pb index 19eb00c3aa32..fbd794ce750f 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.pb and b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.yaml b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.yaml index 112958fa35f7..037791d13d99 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1.StatefulSet.yaml @@ -32,6 +32,9 @@ metadata: uid: "7" spec: minReadySeconds: -1056262432 + persistentVolumeClaimRetentionPolicy: + whenDeleted: /Ş槀墺=Ĉ鳟/d + whenScaled: á podManagementPolicy: Ȍ%ÿ¼璤ňɈ replicas: 896585016 revisionHistoryLimit: -1166275743 @@ -1164,18 +1167,18 @@ spec: phase: ñƍU烈 źfjǰɪ嘞 resizeStatus: ʨɺC`牯 status: - availableReplicas: -1174483980 - collisionCount: 1222237461 + availableReplicas: -1889018254 + collisionCount: 1329525670 conditions: - - lastTransitionTime: "2368-07-30T22:05:09Z" + - lastTransitionTime: "2881-11-22T04:01:01Z" message: "541" reason: "540" - status: 飼蒱鄆&嬜Š&?鳢.ǀŭ瘢颦z - type: 壣V礆á¤拈tY - currentReplicas: 223338274 + status: Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6 + type: '&嬜Š&?鳢.ǀŭ瘢' + currentReplicas: -1929177074 currentRevision: "538" - observedGeneration: -3770279213092072518 - readyReplicas: -175399547 - replicas: -1669166259 + observedGeneration: 5897193066236579633 + readyReplicas: -1190434752 + replicas: 1442750499 updateRevision: "539" - updatedReplicas: -1279136912 + updatedReplicas: -690843774 diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.json b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.json index c5d22102fa05..eda69ff37a0e 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.json +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.json @@ -1693,26 +1693,30 @@ } }, "revisionHistoryLimit": -1166275743, - "minReadySeconds": -1056262432 + "minReadySeconds": -1056262432, + "persistentVolumeClaimRetentionPolicy": { + "whenDeleted": "/Ş槀墺=Ĉ鳟/d", + "whenScaled": "á" + } }, "status": { - "observedGeneration": 1124654959171263717, - "replicas": -1775998279, - "readyReplicas": 233229473, - "currentReplicas": 99917513, - "updatedReplicas": 421164481, + "observedGeneration": 6696165662757041544, + "replicas": -629510776, + "readyReplicas": -69450448, + "currentReplicas": -212409426, + "updatedReplicas": 17761427, "currentRevision": "538", "updateRevision": "539", - "collisionCount": -1137929768, + "collisionCount": -168103644, "conditions": [ { - "type": "/d\u0026蒡榤Ⱦ盜ŭ飼蒱鄆", - "status": "", - "lastTransitionTime": "2358-03-12T13:17:54Z", + "type": "", + "status": "嬜Š\u0026?鳢.ǀŭ瘢颦z疵", + "lastTransitionTime": "2455-07-16T22:37:15Z", "reason": "540", "message": "541" } ], - "availableReplicas": 171558604 + "availableReplicas": 2136031303 } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.pb b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.pb index d0c2eae1dc10..7bd81b2dbb91 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.pb and b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.yaml b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.yaml index 5c5a09f27b5a..0c27d705198f 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta1.StatefulSet.yaml @@ -32,6 +32,9 @@ metadata: uid: "7" spec: minReadySeconds: -1056262432 + persistentVolumeClaimRetentionPolicy: + whenDeleted: /Ş槀墺=Ĉ鳟/d + whenScaled: á podManagementPolicy: Ȍ%ÿ¼璤ňɈ replicas: 896585016 revisionHistoryLimit: -1166275743 @@ -1164,18 +1167,18 @@ spec: phase: ñƍU烈 źfjǰɪ嘞 resizeStatus: ʨɺC`牯 status: - availableReplicas: 171558604 - collisionCount: -1137929768 + availableReplicas: 2136031303 + collisionCount: -168103644 conditions: - - lastTransitionTime: "2358-03-12T13:17:54Z" + - lastTransitionTime: "2455-07-16T22:37:15Z" message: "541" reason: "540" - status: "" - type: /d&蒡榤Ⱦ盜ŭ飼蒱鄆 - currentReplicas: 99917513 + status: 嬜Š&?鳢.ǀŭ瘢颦z疵 + type: "" + currentReplicas: -212409426 currentRevision: "538" - observedGeneration: 1124654959171263717 - readyReplicas: 233229473 - replicas: -1775998279 + observedGeneration: 6696165662757041544 + readyReplicas: -69450448 + replicas: -629510776 updateRevision: "539" - updatedReplicas: 421164481 + updatedReplicas: 17761427 diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.json b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.json index f21e2902b574..a045190e54ce 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.json +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.json @@ -1693,26 +1693,30 @@ } }, "revisionHistoryLimit": -1166275743, - "minReadySeconds": -1056262432 + "minReadySeconds": -1056262432, + "persistentVolumeClaimRetentionPolicy": { + "whenDeleted": "/Ş槀墺=Ĉ鳟/d", + "whenScaled": "á" + } }, "status": { - "observedGeneration": -3770279213092072518, - "replicas": -1669166259, - "readyReplicas": -175399547, - "currentReplicas": 223338274, - "updatedReplicas": -1279136912, + "observedGeneration": 5897193066236579633, + "replicas": 1442750499, + "readyReplicas": -1190434752, + "currentReplicas": -1929177074, + "updatedReplicas": -690843774, "currentRevision": "538", "updateRevision": "539", - "collisionCount": 1222237461, + "collisionCount": 1329525670, "conditions": [ { - "type": "壣V礆á¤拈tY", - "status": "飼蒱鄆\u0026嬜Š\u0026?鳢.ǀŭ瘢颦z", - "lastTransitionTime": "2368-07-30T22:05:09Z", + "type": "\u0026嬜Š\u0026?鳢.ǀŭ瘢", + "status": "Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6", + "lastTransitionTime": "2881-11-22T04:01:01Z", "reason": "540", "message": "541" } ], - "availableReplicas": -1174483980 + "availableReplicas": -1889018254 } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.pb b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.pb index 77331aca867b..a48a4c7fb38c 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.pb and b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.yaml b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.yaml index ee9ce6f6d0b1..cdc54ebad64b 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/apps.v1beta2.StatefulSet.yaml @@ -32,6 +32,9 @@ metadata: uid: "7" spec: minReadySeconds: -1056262432 + persistentVolumeClaimRetentionPolicy: + whenDeleted: /Ş槀墺=Ĉ鳟/d + whenScaled: á podManagementPolicy: Ȍ%ÿ¼璤ňɈ replicas: 896585016 revisionHistoryLimit: -1166275743 @@ -1164,18 +1167,18 @@ spec: phase: ñƍU烈 źfjǰɪ嘞 resizeStatus: ʨɺC`牯 status: - availableReplicas: -1174483980 - collisionCount: 1222237461 + availableReplicas: -1889018254 + collisionCount: 1329525670 conditions: - - lastTransitionTime: "2368-07-30T22:05:09Z" + - lastTransitionTime: "2881-11-22T04:01:01Z" message: "541" reason: "540" - status: 飼蒱鄆&嬜Š&?鳢.ǀŭ瘢颦z - type: 壣V礆á¤拈tY - currentReplicas: 223338274 + status: Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6 + type: '&嬜Š&?鳢.ǀŭ瘢' + currentReplicas: -1929177074 currentRevision: "538" - observedGeneration: -3770279213092072518 - readyReplicas: -175399547 - replicas: -1669166259 + observedGeneration: 5897193066236579633 + readyReplicas: -1190434752 + replicas: 1442750499 updateRevision: "539" - updatedReplicas: -1279136912 + updatedReplicas: -690843774 diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetpersistentvolumeclaimretentionpolicy.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetpersistentvolumeclaimretentionpolicy.go new file mode 100644 index 000000000000..ba01d5d3c10a --- /dev/null +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetpersistentvolumeclaimretentionpolicy.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "k8s.io/api/apps/v1" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use +// with apply. +type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct { + WhenDeleted *v1.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"` + WhenScaled *v1.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"` +} + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with +// apply. +func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} +} + +// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenDeleted field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenDeleted = &value + return b +} + +// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenScaled field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenScaled = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetspec.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetspec.go index ade14180b22d..ee0ed40a5692 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1/statefulsetspec.go @@ -27,15 +27,16 @@ import ( // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // with apply. type StatefulSetSpecApplyConfiguration struct { - Replicas *int32 `json:"replicas,omitempty"` - Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` - Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` - VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` - ServiceName *string `json:"serviceName,omitempty"` - PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` - UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` - MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` + Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` + VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` + ServiceName *string `json:"serviceName,omitempty"` + PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` + UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"` } // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with @@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St b.MinReadySeconds = &value return b } + +// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call. +func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration { + b.PersistentVolumeClaimRetentionPolicy = value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetpersistentvolumeclaimretentionpolicy.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetpersistentvolumeclaimretentionpolicy.go new file mode 100644 index 000000000000..0048724c041c --- /dev/null +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetpersistentvolumeclaimretentionpolicy.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "k8s.io/api/apps/v1beta1" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use +// with apply. +type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct { + WhenDeleted *v1beta1.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"` + WhenScaled *v1beta1.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"` +} + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with +// apply. +func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} +} + +// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenDeleted field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1beta1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenDeleted = &value + return b +} + +// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenScaled field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1beta1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenScaled = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetspec.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetspec.go index befd1f7e0ad2..886433d9ea31 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta1/statefulsetspec.go @@ -27,15 +27,16 @@ import ( // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // with apply. type StatefulSetSpecApplyConfiguration struct { - Replicas *int32 `json:"replicas,omitempty"` - Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` - Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` - VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` - ServiceName *string `json:"serviceName,omitempty"` - PodManagementPolicy *v1beta1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` - UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` - MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` + Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` + VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` + ServiceName *string `json:"serviceName,omitempty"` + PodManagementPolicy *v1beta1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` + UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"` } // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with @@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St b.MinReadySeconds = &value return b } + +// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call. +func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration { + b.PersistentVolumeClaimRetentionPolicy = value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetpersistentvolumeclaimretentionpolicy.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetpersistentvolumeclaimretentionpolicy.go new file mode 100644 index 000000000000..aee27803d3b6 --- /dev/null +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetpersistentvolumeclaimretentionpolicy.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + v1beta2 "k8s.io/api/apps/v1beta2" +) + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use +// with apply. +type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct { + WhenDeleted *v1beta2.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"` + WhenScaled *v1beta2.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"` +} + +// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with +// apply. +func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} +} + +// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenDeleted field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1beta2.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenDeleted = &value + return b +} + +// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WhenScaled field is set to the value of the last call. +func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1beta2.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration { + b.WhenScaled = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetspec.go b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetspec.go index 44044be265c9..08922cea5760 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/apps/v1beta2/statefulsetspec.go @@ -27,15 +27,16 @@ import ( // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // with apply. type StatefulSetSpecApplyConfiguration struct { - Replicas *int32 `json:"replicas,omitempty"` - Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` - Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` - VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` - ServiceName *string `json:"serviceName,omitempty"` - PodManagementPolicy *v1beta2.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` - UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` - MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` + Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` + VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` + ServiceName *string `json:"serviceName,omitempty"` + PodManagementPolicy *v1beta2.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` + UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"` } // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with @@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St b.MinReadySeconds = &value return b } + +// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call. +func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration { + b.PersistentVolumeClaimRetentionPolicy = value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go index 3e4620fda5d0..edf6e7a97f0a 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go @@ -907,12 +907,24 @@ var schemaYAML = typed.YAMLObject(`types: type: scalar: string default: "" +- name: io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy + map: + fields: + - name: whenDeleted + type: + scalar: string + - name: whenScaled + type: + scalar: string - name: io.k8s.api.apps.v1.StatefulSetSpec map: fields: - name: minReadySeconds type: scalar: numeric + - name: persistentVolumeClaimRetentionPolicy + type: + namedType: io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy - name: podManagementPolicy type: scalar: string @@ -1195,12 +1207,24 @@ var schemaYAML = typed.YAMLObject(`types: type: scalar: string default: "" +- name: io.k8s.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy + map: + fields: + - name: whenDeleted + type: + scalar: string + - name: whenScaled + type: + scalar: string - name: io.k8s.api.apps.v1beta1.StatefulSetSpec map: fields: - name: minReadySeconds type: scalar: numeric + - name: persistentVolumeClaimRetentionPolicy + type: + namedType: io.k8s.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy - name: podManagementPolicy type: scalar: string @@ -1681,12 +1705,24 @@ var schemaYAML = typed.YAMLObject(`types: type: scalar: string default: "" +- name: io.k8s.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy + map: + fields: + - name: whenDeleted + type: + scalar: string + - name: whenScaled + type: + scalar: string - name: io.k8s.api.apps.v1beta2.StatefulSetSpec map: fields: - name: minReadySeconds type: scalar: numeric + - name: persistentVolumeClaimRetentionPolicy + type: + namedType: io.k8s.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy - name: podManagementPolicy type: scalar: string diff --git a/staging/src/k8s.io/client-go/applyconfigurations/utils.go b/staging/src/k8s.io/client-go/applyconfigurations/utils.go index dc6d3835ce41..195b3ef79458 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/utils.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/utils.go @@ -189,6 +189,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &applyconfigurationsappsv1.StatefulSetApplyConfiguration{} case appsv1.SchemeGroupVersion.WithKind("StatefulSetCondition"): return &applyconfigurationsappsv1.StatefulSetConditionApplyConfiguration{} + case appsv1.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"): + return &applyconfigurationsappsv1.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} case appsv1.SchemeGroupVersion.WithKind("StatefulSetSpec"): return &applyconfigurationsappsv1.StatefulSetSpecApplyConfiguration{} case appsv1.SchemeGroupVersion.WithKind("StatefulSetStatus"): @@ -219,6 +221,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &applyconfigurationsappsv1beta1.StatefulSetApplyConfiguration{} case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetCondition"): return &applyconfigurationsappsv1beta1.StatefulSetConditionApplyConfiguration{} + case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"): + return &applyconfigurationsappsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetSpec"): return &applyconfigurationsappsv1beta1.StatefulSetSpecApplyConfiguration{} case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetStatus"): @@ -269,6 +273,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &appsv1beta2.StatefulSetApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("StatefulSetCondition"): return &appsv1beta2.StatefulSetConditionApplyConfiguration{} + case v1beta2.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"): + return &appsv1beta2.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("StatefulSetSpec"): return &appsv1beta2.StatefulSetSpecApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("StatefulSetStatus"): diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index 1f0bd7627429..6a347e171b17 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -20,6 +20,8 @@ import ( "context" "encoding/json" "fmt" + "regexp" + "strconv" "strings" "sync" "time" @@ -1142,6 +1144,7 @@ var _ = SIGDescribe("StatefulSet", func() { appTester.run() }) }) + // Make sure minReadySeconds is honored // Don't mark it as conformance yet ginkgo.It("MinReadySeconds should be honored when enabled", func() { @@ -1202,6 +1205,144 @@ var _ = SIGDescribe("StatefulSet", func() { framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 2, out) } }) + + ginkgo.Describe("Non-retain StatefulSetPersistentVolumeClaimPolicy [Feature:StatefulSetAutoDeletePVC]", func() { + ssName := "ss" + labels := map[string]string{ + "foo": "bar", + "baz": "blah", + } + headlessSvcName := "test" + var statefulPodMounts, podMounts []v1.VolumeMount + var ss *appsv1.StatefulSet + + ginkgo.BeforeEach(func() { + statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}} + podMounts = []v1.VolumeMount{{Name: "home", MountPath: "/home"}} + ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, statefulPodMounts, podMounts, labels) + + ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns) + headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels) + _, err := c.CoreV1().Services(ns).Create(context.TODO(), headlessService, metav1.CreateOptions{}) + framework.ExpectNoError(err) + }) + + ginkgo.AfterEach(func() { + if ginkgo.CurrentGinkgoTestDescription().Failed { + framework.DumpDebugInfo(c, ns) + } + framework.Logf("Deleting all statefulset in ns %v", ns) + e2estatefulset.DeleteAllStatefulSets(c, ns) + }) + + ginkgo.It("should delete PVCs with a WhenDeleted policy", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist with their owner refs") + err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false) + framework.ExpectNoError(err) + + ginkgo.By("Deleting stateful set " + ss.Name) + err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Verifying PVCs deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs with a OnScaledown policy", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist") + err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2}) + framework.ExpectNoError(err) + + ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") + ss, err = e2estatefulset.Scale(c, ss, 1) + framework.ExpectNoError(err) + + ginkgo.By("Verifying all but one PVC deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{0}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs after adopting pod (WhenDeleted)", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist with their owner refs") + err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false) + framework.ExpectNoError(err) + + ginkgo.By("Orphaning the 3rd pod") + patch, err := json.Marshal(metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{}, + }) + framework.ExpectNoError(err, "Could not Marshal JSON for patch payload") + _, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "") + framework.ExpectNoError(err, "Could not patch payload") + + ginkgo.By("Deleting stateful set " + ss.Name) + err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Verifying PVCs deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs after adopting pod (WhenScaled) [Feature:StatefulSetAutoDeletePVC]", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist") + err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2}) + framework.ExpectNoError(err) + + ginkgo.By("Orphaning the 3rd pod") + patch, err := json.Marshal(metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{}, + }) + framework.ExpectNoError(err, "Could not Marshal JSON for patch payload") + _, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "") + framework.ExpectNoError(err, "Could not patch payload") + + ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") + ss, err = e2estatefulset.Scale(c, ss, 1) + framework.ExpectNoError(err) + + ginkgo.By("Verifying all but one PVC deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{0}) + framework.ExpectNoError(err) + }) + }) }) func kubectlExecWithRetries(ns string, args ...string) (out string) { @@ -1645,3 +1786,111 @@ func getStatefulSet(c clientset.Interface, namespace, name string) *appsv1.State } return ss } + +// verifyStatefulSetPVCsExist confirms that exactly the PVCs for ss with the specified ids exist. This polls until the situation occurs, an error happens, or until timeout (in the latter case an error is also returned). Beware that this cannot tell if a PVC will be deleted at some point in the future, so if used to confirm that no PVCs are deleted, the caller should wait for some event giving the PVCs a reasonable chance to be deleted, before calling this function. +func verifyStatefulSetPVCsExist(c clientset.Interface, ss *appsv1.StatefulSet, claimIds []int) error { + idSet := map[int]struct{}{} + for _, id := range claimIds { + idSet[id] = struct{}{} + } + return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) { + pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) + if err != nil { + framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) + return false, nil + } + for _, claim := range ss.Spec.VolumeClaimTemplates { + pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name)) + seenPVCs := map[int]struct{}{} + for _, pvc := range pvcList.Items { + matches := pvcNameRE.FindStringSubmatch(pvc.Name) + if len(matches) != 2 { + continue + } + ordinal, err := strconv.ParseInt(matches[1], 10, 32) + if err != nil { + framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err) + return false, err + } + if _, found := idSet[int(ordinal)]; !found { + return false, nil // Retry until the PVCs are consistent. + } else { + seenPVCs[int(ordinal)] = struct{}{} + } + } + if len(seenPVCs) != len(idSet) { + framework.Logf("Found %d of %d PVCs", len(seenPVCs), len(idSet)) + return false, nil // Retry until the PVCs are consistent. + } + } + return true, nil + }) +} + +// verifyStatefulSetPVCsExistWithOwnerRefs works as verifyStatefulSetPVCsExist, but also waits for the ownerRefs to match. +func verifyStatefulSetPVCsExistWithOwnerRefs(c clientset.Interface, ss *appsv1.StatefulSet, claimIndicies []int, wantSetRef, wantPodRef bool) error { + indexSet := map[int]struct{}{} + for _, id := range claimIndicies { + indexSet[id] = struct{}{} + } + set := getStatefulSet(c, ss.Namespace, ss.Name) + setUID := set.GetUID() + if setUID == "" { + framework.Failf("Statefulset %s mising UID", ss.Name) + } + return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) { + pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) + if err != nil { + framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) + return false, nil + } + for _, claim := range ss.Spec.VolumeClaimTemplates { + pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name)) + seenPVCs := map[int]struct{}{} + for _, pvc := range pvcList.Items { + matches := pvcNameRE.FindStringSubmatch(pvc.Name) + if len(matches) != 2 { + continue + } + ordinal, err := strconv.ParseInt(matches[1], 10, 32) + if err != nil { + framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err) + return false, err + } + if _, found := indexSet[int(ordinal)]; !found { + framework.Logf("Unexpected, retrying") + return false, nil // Retry until the PVCs are consistent. + } + var foundSetRef, foundPodRef bool + for _, ref := range pvc.GetOwnerReferences() { + if ref.Kind == "StatefulSet" && ref.UID == setUID { + foundSetRef = true + } + if ref.Kind == "Pod" { + podName := fmt.Sprintf("%s-%d", ss.Name, ordinal) + pod, err := c.CoreV1().Pods(ss.Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + framework.Logf("Pod %s not found, retrying (%v)", podName, err) + return false, nil + } + podUID := pod.GetUID() + if podUID == "" { + framework.Failf("Pod %s is missing UID", pod.Name) + } + if ref.UID == podUID { + foundPodRef = true + } + } + } + if foundSetRef == wantSetRef && foundPodRef == wantPodRef { + seenPVCs[int(ordinal)] = struct{}{} + } + } + if len(seenPVCs) != len(indexSet) { + framework.Logf("Only %d PVCs, retrying", len(seenPVCs)) + return false, nil // Retry until the PVCs are consistent. + } + } + return true, nil + }) +}