Skip to content

Commit

Permalink
Merge pull request #98515 from lala123912/huge_page
Browse files Browse the repository at this point in the history 
Add request value verification for hugepage
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Mar 6, 2021
2 parents 56bcd56 + e162fcc commit 4e95e1d
Show file tree
Hide file tree
Showing 8 changed files with 530 additions and 9 deletions.
52 changes: 52 additions & 0 deletions pkg/api/pod/util.go
View file
Expand Up @@ -345,6 +345,52 @@ func usesMultipleHugePageResources(podSpec *api.PodSpec) bool {
return len(hugePageResources) > 1
}

func checkContainerUseIndivisibleHugePagesValues(container api.Container) bool {
for resourceName, quantity := range container.Resources.Limits {
if helper.IsHugePageResourceName(resourceName) {
if !helper.IsHugePageResourceValueDivisible(resourceName, quantity) {
return true
}
}
}

for resourceName, quantity := range container.Resources.Requests {
if helper.IsHugePageResourceName(resourceName) {
if !helper.IsHugePageResourceValueDivisible(resourceName, quantity) {
return true
}
}
}

return false
}

// usesIndivisibleHugePagesValues returns true if the one of the containers uses non-integer multiple
// of huge page unit size
func usesIndivisibleHugePagesValues(podSpec *api.PodSpec) bool {
foundIndivisibleHugePagesValue := false
VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
if checkContainerUseIndivisibleHugePagesValues(*c) {
foundIndivisibleHugePagesValue = true
}
return !foundIndivisibleHugePagesValue // continue visiting if we haven't seen an invalid value yet
})

if foundIndivisibleHugePagesValue {
return true
}

for resourceName, quantity := range podSpec.Overhead {
if helper.IsHugePageResourceName(resourceName) {
if !helper.IsHugePageResourceValueDivisible(resourceName, quantity) {
return true
}
}
}

return false
}

// GetValidationOptionsFromPodSpecAndMeta returns validation options based on pod specs and metadata
func GetValidationOptionsFromPodSpecAndMeta(podSpec, oldPodSpec *api.PodSpec, podMeta, oldPodMeta *metav1.ObjectMeta) apivalidation.PodValidationOptions {
// default pod validation options based on feature gate
Expand All @@ -354,6 +400,8 @@ func GetValidationOptionsFromPodSpecAndMeta(podSpec, oldPodSpec *api.PodSpec, po
// Allow pod spec to use hugepages in downward API if feature is enabled
AllowDownwardAPIHugePages: utilfeature.DefaultFeatureGate.Enabled(features.DownwardAPIHugePages),
AllowInvalidPodDeletionCost: !utilfeature.DefaultFeatureGate.Enabled(features.PodDeletionCost),
// Do not allow pod spec to use non-integer multiple of huge page unit size default
AllowIndivisibleHugePagesValues: false,
}

if oldPodSpec != nil {
Expand All @@ -368,12 +416,16 @@ func GetValidationOptionsFromPodSpecAndMeta(podSpec, oldPodSpec *api.PodSpec, po
return !opts.AllowDownwardAPIHugePages
})
}

// if old spec used non-integer multiple of huge page unit size, we must allow it
opts.AllowIndivisibleHugePagesValues = usesIndivisibleHugePagesValues(oldPodSpec)
}
if oldPodMeta != nil && !opts.AllowInvalidPodDeletionCost {
// This is an update, so validate only if the existing object was valid.
_, err := helper.GetDeletionCostFromPodAnnotations(oldPodMeta.Annotations)
opts.AllowInvalidPodDeletionCost = err != nil
}

return opts
}

Expand Down
15 changes: 15 additions & 0 deletions pkg/apis/core/helper/helpers.go
View file
Expand Up @@ -39,6 +39,21 @@ func IsHugePageResourceName(name core.ResourceName) bool {
return strings.HasPrefix(string(name), core.ResourceHugePagesPrefix)
}

// IsHugePageResourceValueDivisible returns true if the resource value of storage is
// integer multiple of page size.
func IsHugePageResourceValueDivisible(name core.ResourceName, quantity resource.Quantity) bool {
pageSize, err := HugePageSizeFromResourceName(name)
if err != nil {
return false
}

if pageSize.Sign() <= 0 || pageSize.MilliValue()%int64(1000) != int64(0) {
return false
}

return quantity.Value()%pageSize.Value() == 0
}

// IsQuotaHugePageResourceName returns true if the resource name has the quota
// related huge page resource prefix.
func IsQuotaHugePageResourceName(name core.ResourceName) bool {
Expand Down
54 changes: 54 additions & 0 deletions pkg/apis/core/helper/helpers_test.go
View file
Expand Up @@ -211,6 +211,60 @@ func TestIsHugePageResourceName(t *testing.T) {
}
}

func TestIsHugePageResourceValueDivisible(t *testing.T) {
testCases := []struct {
name core.ResourceName
quantity resource.Quantity
result bool
}{
{
name: core.ResourceName("hugepages-2Mi"),
quantity: resource.MustParse("4Mi"),
result: true,
},
{
name: core.ResourceName("hugepages-2Mi"),
quantity: resource.MustParse("5Mi"),
result: false,
},
{
name: core.ResourceName("hugepages-1Gi"),
quantity: resource.MustParse("2Gi"),
result: true,
},
{
name: core.ResourceName("hugepages-1Gi"),
quantity: resource.MustParse("2.1Gi"),
result: false,
},
{
name: core.ResourceName("hugepages-1Mi"),
quantity: resource.MustParse("2.1Mi"),
result: false,
},
{
name: core.ResourceName("hugepages-64Ki"),
quantity: resource.MustParse("128Ki"),
result: true,
},
{
name: core.ResourceName("hugepages-"),
quantity: resource.MustParse("128Ki"),
result: false,
},
{
name: core.ResourceName("hugepages"),
quantity: resource.MustParse("128Ki"),
result: false,
},
}
for _, testCase := range testCases {
if testCase.result != IsHugePageResourceValueDivisible(testCase.name, testCase.quantity) {
t.Errorf("resource: %v storage:%v expected result: %v", testCase.name, testCase.quantity, testCase.result)
}
}
}

func TestHugePageResourceName(t *testing.T) {
testCases := []struct {
pageSize resource.Quantity
Expand Down
30 changes: 25 additions & 5 deletions pkg/apis/core/validation/validation.go
View file
Expand Up @@ -301,9 +301,9 @@ func ValidateRuntimeClassName(name string, fldPath *field.Path) field.ErrorList
}

// validateOverhead can be used to check whether the given Overhead is valid.
func validateOverhead(overhead core.ResourceList, fldPath *field.Path) field.ErrorList {
func validateOverhead(overhead core.ResourceList, fldPath *field.Path, opts PodValidationOptions) field.ErrorList {
// reuse the ResourceRequirements validation logic
return ValidateResourceRequirements(&core.ResourceRequirements{Limits: overhead}, fldPath)
return ValidateResourceRequirements(&core.ResourceRequirements{Limits: overhead}, fldPath, opts)
}

// Validates that given value is not negative.
Expand Down Expand Up @@ -2889,7 +2889,7 @@ func validateContainers(containers []core.Container, isInitContainers bool, volu
allErrs = append(allErrs, ValidateVolumeMounts(ctr.VolumeMounts, volDevices, volumes, &ctr, idxPath.Child("volumeMounts"))...)
allErrs = append(allErrs, ValidateVolumeDevices(ctr.VolumeDevices, volMounts, volumes, idxPath.Child("volumeDevices"))...)
allErrs = append(allErrs, validatePullPolicy(ctr.ImagePullPolicy, idxPath.Child("imagePullPolicy"))...)
allErrs = append(allErrs, ValidateResourceRequirements(&ctr.Resources, idxPath.Child("resources"))...)
allErrs = append(allErrs, ValidateResourceRequirements(&ctr.Resources, idxPath.Child("resources"), opts)...)
allErrs = append(allErrs, ValidateSecurityContext(ctr.SecurityContext, idxPath.Child("securityContext"))...)
}

Expand Down Expand Up @@ -3202,6 +3202,8 @@ type PodValidationOptions struct {
AllowDownwardAPIHugePages bool
// Allow invalid pod-deletion-cost annotation value for backward compatibility.
AllowInvalidPodDeletionCost bool
// Allow pod spec to use non-integer multiple of huge page unit size
AllowIndivisibleHugePagesValues bool
}

// ValidatePodSingleHugePageResources checks if there are multiple huge
Expand Down Expand Up @@ -3375,7 +3377,7 @@ func ValidatePodSpec(spec *core.PodSpec, podMeta *metav1.ObjectMeta, fldPath *fi
}

if spec.Overhead != nil {
allErrs = append(allErrs, validateOverhead(spec.Overhead, fldPath.Child("overhead"))...)
allErrs = append(allErrs, validateOverhead(spec.Overhead, fldPath.Child("overhead"), opts)...)
}

return allErrs
Expand Down Expand Up @@ -5336,7 +5338,7 @@ func validateBasicResource(quantity resource.Quantity, fldPath *field.Path) fiel
}

// Validates resource requirement spec.
func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPath *field.Path) field.ErrorList {
func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPath *field.Path, opts PodValidationOptions) field.ErrorList {
allErrs := field.ErrorList{}
limPath := fldPath.Child("limits")
reqPath := fldPath.Child("requests")
Expand All @@ -5356,6 +5358,9 @@ func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPa

if helper.IsHugePageResourceName(resourceName) {
limContainsHugePages = true
if err := validateResourceQuantityHugePageValue(resourceName, quantity, opts); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, quantity.String(), err.Error()))
}
}

if supportedQoSComputeResources.Has(string(resourceName)) {
Expand Down Expand Up @@ -5383,6 +5388,9 @@ func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPa
}
if helper.IsHugePageResourceName(resourceName) {
reqContainsHugePages = true
if err := validateResourceQuantityHugePageValue(resourceName, quantity, opts); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, quantity.String(), err.Error()))
}
}
if supportedQoSComputeResources.Has(string(resourceName)) {
reqContainsCPUOrMemory = true
Expand All @@ -5396,6 +5404,18 @@ func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPa
return allErrs
}

func validateResourceQuantityHugePageValue(name core.ResourceName, quantity resource.Quantity, opts PodValidationOptions) error {
if !helper.IsHugePageResourceName(name) {
return nil
}

if !opts.AllowIndivisibleHugePagesValues && !helper.IsHugePageResourceValueDivisible(name, quantity) {
return fmt.Errorf("%s is not positive integer multiple of %s", quantity.String(), name)
}

return nil
}

// validateResourceQuotaScopes ensures that each enumerated hard resource constraint is valid for set of scopes
func validateResourceQuotaScopes(resourceQuotaSpec *core.ResourceQuotaSpec, opts ResourceQuotaValidationOptions, fld *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
Expand Down
89 changes: 86 additions & 3 deletions pkg/apis/core/validation/validation_test.go
View file
Expand Up @@ -4446,7 +4446,7 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) {
resource.BinarySI),
},
}
if errs := ValidateResourceRequirements(&containerLimitCase, field.NewPath("resources")); len(errs) != 0 {
if errs := ValidateResourceRequirements(&containerLimitCase, field.NewPath("resources"), PodValidationOptions{}); len(errs) != 0 {
t.Errorf("expected success: %v", errs)
}
}
Expand Down Expand Up @@ -16571,7 +16571,7 @@ func TestValidateOverhead(t *testing.T) {
},
}
for _, tc := range successCase {
if errs := validateOverhead(tc.overhead, field.NewPath("overheads")); len(errs) != 0 {
if errs := validateOverhead(tc.overhead, field.NewPath("overheads"), PodValidationOptions{}); len(errs) != 0 {
t.Errorf("%q unexpected error: %v", tc.Name, errs)
}
}
Expand All @@ -16588,7 +16588,7 @@ func TestValidateOverhead(t *testing.T) {
},
}
for _, tc := range errorCase {
if errs := validateOverhead(tc.overhead, field.NewPath("resources")); len(errs) == 0 {
if errs := validateOverhead(tc.overhead, field.NewPath("resources"), PodValidationOptions{}); len(errs) == 0 {
t.Errorf("%q expected error", tc.Name)
}
}
Expand Down Expand Up @@ -17248,3 +17248,86 @@ func TestValidatePodTemplateSpecSeccomp(t *testing.T) {
asserttestify.Equal(t, test.expectedErr, err, "TestCase[%d]: %s", i, test.description)
}
}

func TestValidateResourceRequirements(t *testing.T) {
path := field.NewPath("resources")
tests := []struct {
name string
requirements core.ResourceRequirements
opts PodValidationOptions
}{
{
name: "limits and requests of hugepage resource are equal",
requirements: core.ResourceRequirements{
Limits: core.ResourceList{
core.ResourceCPU: resource.MustParse("10"),
core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"),
},
Requests: core.ResourceList{
core.ResourceCPU: resource.MustParse("10"),
core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"),
},
},
opts: PodValidationOptions{},
},
{
name: "limits and requests of memory resource are equal",
requirements: core.ResourceRequirements{
Limits: core.ResourceList{
core.ResourceMemory: resource.MustParse("2Mi"),
},
Requests: core.ResourceList{
core.ResourceMemory: resource.MustParse("2Mi"),
},
},
opts: PodValidationOptions{},
},
{
name: "limits and requests of cpu resource are equal",
requirements: core.ResourceRequirements{
Limits: core.ResourceList{
core.ResourceCPU: resource.MustParse("10"),
},
Requests: core.ResourceList{
core.ResourceCPU: resource.MustParse("10"),
},
},
opts: PodValidationOptions{},
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if errs := ValidateResourceRequirements(&tc.requirements, path, tc.opts); len(errs) != 0 {
t.Errorf("unexpected errors: %v", errs)
}
})
}

errTests := []struct {
name string
requirements core.ResourceRequirements
opts PodValidationOptions
}{
{
name: "hugepage resource without cpu or memory",
requirements: core.ResourceRequirements{
Limits: core.ResourceList{
core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"),
},
Requests: core.ResourceList{
core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"),
},
},
opts: PodValidationOptions{},
},
}

for _, tc := range errTests {
t.Run(tc.name, func(t *testing.T) {
if errs := ValidateResourceRequirements(&tc.requirements, path, tc.opts); len(errs) == 0 {
t.Error("expected errors")
}
})
}
}
3 changes: 2 additions & 1 deletion pkg/apis/node/validation/validation.go
View file
Expand Up @@ -54,7 +54,8 @@ func ValidateRuntimeClassUpdate(new, old *node.RuntimeClass) field.ErrorList {

func validateOverhead(overhead *node.Overhead, fldPath *field.Path) field.ErrorList {
// reuse the ResourceRequirements validation logic
return corevalidation.ValidateResourceRequirements(&core.ResourceRequirements{Limits: overhead.PodFixed}, fldPath)
return corevalidation.ValidateResourceRequirements(&core.ResourceRequirements{Limits: overhead.PodFixed}, fldPath,
corevalidation.PodValidationOptions{})
}

func validateScheduling(s *node.Scheduling, fldPath *field.Path) field.ErrorList {
Expand Down

0 comments on commit 4e95e1d

Please sign in to comment.