Skip to content

Commit

Permalink
Merge pull request #99728 from mattcary/control
Browse files Browse the repository at this point in the history 
StatefulSet PVC auto-delete implementation
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Nov 18, 2021
2 parents 1304dfd + 53b3a6c commit b8af116
Show file tree
Hide file tree
Showing 59 changed files with 4,949 additions and 1,132 deletions.
18 changes: 18 additions & 0 deletions api/openapi-spec/swagger.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 18 additions & 0 deletions api/openapi-spec/v3/apis__apps__v1_openapi.json
View file
Expand Up @@ -8822,6 +8822,20 @@
}
]
},
"io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy": {
"description": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"type": "object",
"properties": {
"whenDeleted": {
"description": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"type": "string"
},
"whenScaled": {
"description": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
"type": "string"
}
}
},
"io.k8s.api.apps.v1.StatefulSetSpec": {
"description": "A StatefulSetSpec is the specification of a StatefulSet.",
"type": "object",
Expand All @@ -8836,6 +8850,10 @@
"type": "integer",
"format": "int32"
},
"persistentVolumeClaimRetentionPolicy": {
"description": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional",
"$ref": "#/components/schemas/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy"
},
"podManagementPolicy": {
"description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.",
"type": "string",
Expand Down
11 changes: 11 additions & 0 deletions pkg/apis/apps/fuzzer/fuzzer.go
View file
Expand Up @@ -40,6 +40,17 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
if len(s.Spec.UpdateStrategy.Type) == 0 {
s.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType
}
if s.Spec.PersistentVolumeClaimRetentionPolicy == nil {
policies := []apps.PersistentVolumeClaimRetentionPolicyType{
apps.RetainPersistentVolumeClaimRetentionPolicyType,
apps.DeletePersistentVolumeClaimRetentionPolicyType,
}
choice := int32(c.Rand.Int31())
s.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: policies[choice&1],
WhenScaled: policies[(choice>>1)&1],
}
}
if s.Spec.RevisionHistoryLimit == nil {
s.Spec.RevisionHistoryLimit = new(int32)
*s.Spec.RevisionHistoryLimit = 10
Expand Down
40 changes: 40 additions & 0 deletions pkg/apis/apps/types.go
View file
Expand Up @@ -97,6 +97,40 @@ type RollingUpdateStatefulSetStrategy struct {
Partition int32
}

// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
// deleted or scaled down.
type PersistentVolumeClaimRetentionPolicyType string

const (
// RetainPersistentVolumeClaimRetentionPolicyType is the default
// PersistentVolumeClaimRetentionPolicy and specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will not be deleted.
RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
// DeletePersistentVolumeClaimRetentionPolicyType specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will be deleted in the scenario specified in
// StatefulSetPersistentVolumeClaimPolicy.
DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
)

// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
WhenDeleted PersistentVolumeClaimRetentionPolicyType
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
WhenScaled PersistentVolumeClaimRetentionPolicyType
}

// A StatefulSetSpec is the specification of a StatefulSet.
type StatefulSetSpec struct {
// Replicas is the desired number of replicas of the given Template.
Expand Down Expand Up @@ -164,6 +198,12 @@ type StatefulSetSpec struct {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional
MinReadySeconds int32

// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy
}

// StatefulSetStatus represents the current state of a StatefulSet.
Expand Down
14 changes: 14 additions & 0 deletions pkg/apis/apps/v1/defaults.go
View file
Expand Up @@ -20,6 +20,8 @@ import (
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
)

func addDefaultingFuncs(scheme *runtime.Scheme) error {
Expand Down Expand Up @@ -117,6 +119,18 @@ func SetDefaults_StatefulSet(obj *appsv1.StatefulSet) {
*obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0
}

if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1.RetainPersistentVolumeClaimRetentionPolicyType
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1.RetainPersistentVolumeClaimRetentionPolicyType
}
}

if obj.Spec.Replicas == nil {
obj.Spec.Replicas = new(int32)
*obj.Spec.Replicas = 1
Expand Down
176 changes: 158 additions & 18 deletions pkg/apis/apps/v1/defaults_test.go
View file
Expand Up @@ -27,10 +27,13 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/api/legacyscheme"
_ "k8s.io/kubernetes/pkg/apis/apps/install"
. "k8s.io/kubernetes/pkg/apis/apps/v1"
_ "k8s.io/kubernetes/pkg/apis/core/install"
"k8s.io/kubernetes/pkg/features"
utilpointer "k8s.io/utils/pointer"
)

Expand Down Expand Up @@ -193,10 +196,13 @@ func TestSetDefaultStatefulSet(t *testing.T) {
}

tests := []struct {
original *appsv1.StatefulSet
expected *appsv1.StatefulSet
name string
original *appsv1.StatefulSet
expected *appsv1.StatefulSet
enablePVCDeletionPolicy bool
}{
{ // labels and default update strategy
{
name: "labels and default update strategy",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
Expand All @@ -221,7 +227,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
},
},
},
{ // Alternate update strategy
{
name: "Alternate update strategy",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
Expand All @@ -246,7 +253,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
},
},
},
{ // Parallel pod management policy.
{
name: "Parallel pod management policy",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
Expand All @@ -272,7 +280,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
},
},
},
{ // UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set
{
name: "UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
Expand Down Expand Up @@ -302,20 +311,151 @@ func TestSetDefaultStatefulSet(t *testing.T) {
},
},
},
{
name: "PVC delete policy enabled, no policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy enabled, with scaledown policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy disabled, with set deletion policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy disabled, with policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: false,
},
}

for i, test := range tests {
original := test.original
expected := test.expected
obj2 := roundTrip(t, runtime.Object(original))
got, ok := obj2.(*appsv1.StatefulSet)
if !ok {
t.Errorf("(%d) unexpected object: %v", i, got)
t.FailNow()
}
if !apiequality.Semantic.DeepEqual(got.Spec, expected.Spec) {
t.Errorf("(%d) got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", i, got.Spec, expected.Spec)
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, test.enablePVCDeletionPolicy)()

obj2 := roundTrip(t, runtime.Object(test.original))
got, ok := obj2.(*appsv1.StatefulSet)
if !ok {
t.Errorf("unexpected object: %v", got)
t.FailNow()
}
if !apiequality.Semantic.DeepEqual(got.Spec, test.expected.Spec) {
t.Errorf("got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", got.Spec, test.expected.Spec)
}
})
}
}

Expand Down

0 comments on commit b8af116

Please sign in to comment.