Skip to content

Commit

Permalink
Map PV access modes to CSI access modes based on driver capability
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrishenzie
chrishenzie committed Jun 3, 2021
1 parent 970d0f7 commit 421bd42
Show file tree
Hide file tree
Showing 21 changed files with 493 additions and 92 deletions.
11 changes: 7 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ require (
golang.org/x/text v0.3.5 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210317182105-75c7a8546eb9 // indirect
google.golang.org/grpc v1.36.0
google.golang.org/grpc v1.37.0
google.golang.org/protobuf v1.26.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.21.0
k8s.io/api v0.21.1
k8s.io/apimachinery v0.21.0
k8s.io/apiserver v0.21.0
k8s.io/client-go v0.21.0
k8s.io/component-base v0.21.0
k8s.io/client-go v0.21.1
k8s.io/component-base v0.21.1
k8s.io/component-helpers v0.21.0
k8s.io/csi-translation-lib v0.21.0
k8s.io/klog/v2 v2.8.0
Expand All @@ -43,6 +43,9 @@ require (
sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.3.0
)

// go get -d github.com/chrishenzie/csi-lib-utils@single-node-access-modes
replace github.com/kubernetes-csi/csi-lib-utils => github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af

// go get -d github.com/chrishenzie/kubernetes/staging/src/k8s.io/api@read-write-once-pod-access-mode
replace k8s.io/api => github.com/chrishenzie/kubernetes/staging/src/k8s.io/api v0.0.0-20210507180302-a29b4b67ec78

Expand Down
11 changes: 5 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af h1:rNyFwAyxVYHRhVfJAOdTpyCdk9GAORCccWn0UwZpxug=
github.com/chrishenzie/csi-lib-utils v0.9.2-0.20210603000358-d0686c7b81af/go.mod h1:D0gR5OCNhxbA7T54s7rcwokSueAUE/G7JjrLsZ8jI0M=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/api v0.0.0-20210507180302-a29b4b67ec78 h1:vDfCcW8hb8tcfyBxS2bPKirFbvXII6uuVXgH8+JZLbo=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/api v0.0.0-20210507180302-a29b4b67ec78/go.mod h1:DKjoC7WTLvupppdmb5jEvRDPQENLZqz/stEUs19TOOc=
github.com/chrishenzie/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20210507180302-a29b4b67ec78 h1:Eqob+otS8MBDj3YuC46nQyl/Da0hrioQBSjmjO1cQmc=
Expand Down Expand Up @@ -144,7 +146,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
Expand Down Expand Up @@ -330,8 +332,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
github.com/kubernetes-csi/csi-test/v4 v4.0.2 h1:MNj94SFHOGK6lOy+yDgxI+zlFWaPcgByqBH3JZZGyZI=
github.com/kubernetes-csi/csi-test/v4 v4.0.2/go.mod h1:z3FYigjLFAuzmFzKdHQr8gUPm5Xr4Du2twKcxfys0eI=
github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0 h1:OYDCOjVcx/5wNzlZ/At8otRibUlw0T6R0xOD31f32bw=
Expand Down Expand Up @@ -868,12 +868,11 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c=
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
71 changes: 44 additions & 27 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"time"

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/kubernetes-csi/csi-lib-utils/accessmodes"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
Expand Down Expand Up @@ -451,42 +452,59 @@ func getAccessTypeMount(fsType string, mountFlags []string) *csi.VolumeCapabilit
}
}

func getAccessMode(pvcAccessMode v1.PersistentVolumeAccessMode) *csi.VolumeCapability_AccessMode {
switch pvcAccessMode {
case v1.ReadWriteOnce:
return &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
}
case v1.ReadWriteMany:
return &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
}
case v1.ReadOnlyMany:
return &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
}
default:
return nil
}
}

func getVolumeCapability(
claim *v1.PersistentVolumeClaim,
sc *storagev1.StorageClass,
pvcAccessMode v1.PersistentVolumeAccessMode,
fsType string,
) *csi.VolumeCapability {
supportsSingleNodeMultiWriter bool,
) (*csi.VolumeCapability, error) {
var accessMode csi.VolumeCapability_AccessMode_Mode
var err error
if supportsSingleNodeMultiWriter {
accessMode, err = accessmodes.ToSingleNodeMultiWriterCapableCSIAccessMode([]v1.PersistentVolumeAccessMode{pvcAccessMode})
} else {
accessMode, err = accessmodes.ToCSIAccessMode([]v1.PersistentVolumeAccessMode{pvcAccessMode})
}
if err != nil {
return nil, err
}

if util.CheckPersistentVolumeClaimModeBlock(claim) {
return &csi.VolumeCapability{
AccessType: getAccessTypeBlock(),
AccessMode: getAccessMode(pvcAccessMode),
}
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: accessMode,
},
}, nil
}
return &csi.VolumeCapability{
AccessType: getAccessTypeMount(fsType, sc.MountOptions),
AccessMode: getAccessMode(pvcAccessMode),
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: accessMode,
},
}, nil
}

func (p *csiProvisioner) getVolumeCapabilities(
claim *v1.PersistentVolumeClaim,
sc *storagev1.StorageClass,
fsType string,
) ([]*csi.VolumeCapability, error) {
supportsSingleNodeMultiWriter := false
if p.controllerCapabilities[csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER] {
supportsSingleNodeMultiWriter = true
}

volumeCaps := make([]*csi.VolumeCapability, 0)
for _, pvcAccessMode := range claim.Spec.AccessModes {
volumeCap, err := getVolumeCapability(claim, sc, pvcAccessMode, fsType, supportsSingleNodeMultiWriter)
if err != nil {
return []*csi.VolumeCapability{}, err
}
volumeCaps = append(volumeCaps, volumeCap)
}
return volumeCaps, nil
}

type prepareProvisionResult struct {
Expand Down Expand Up @@ -581,10 +599,9 @@ func (p *csiProvisioner) prepareProvision(ctx context.Context, claim *v1.Persist
capacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
volSizeBytes := capacity.Value()

// Get access mode
volumeCaps := make([]*csi.VolumeCapability, 0)
for _, pvcAccessMode := range claim.Spec.AccessModes {
volumeCaps = append(volumeCaps, getVolumeCapability(claim, sc, pvcAccessMode, fsType))
volumeCaps, err := p.getVolumeCapabilities(claim, sc, fsType)
if err != nil {
return nil, controller.ProvisioningFinished, err
}

// Create a CSI CreateVolumeRequest and Response
Expand Down
167 changes: 145 additions & 22 deletions pkg/controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,15 @@ func provisionCapabilities() (rpc.PluginCapabilitySet, rpc.ControllerCapabilityS
}
}

func provisionWithSingleNodeMultiWriterCapabilities() (rpc.PluginCapabilitySet, rpc.ControllerCapabilitySet) {
return rpc.PluginCapabilitySet{
csi.PluginCapability_Service_CONTROLLER_SERVICE: true,
}, rpc.ControllerCapabilitySet{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME: true,
csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER: true,
}
}

func provisionFromSnapshotCapabilities() (rpc.PluginCapabilitySet, rpc.ControllerCapabilitySet) {
return rpc.PluginCapabilitySet{
csi.PluginCapability_Service_CONTROLLER_SERVICE: true,
Expand Down Expand Up @@ -805,26 +814,27 @@ func TestGetSecretReference(t *testing.T) {
}

type provisioningTestcase struct {
capacity int64 // if zero, default capacity, otherwise available bytes
volOpts controller.ProvisionOptions
notNilSelector bool
makeVolumeNameErr bool
getSecretRefErr bool
getCredentialsErr bool
volWithLessCap bool
volWithZeroCap bool
expectedPVSpec *pvSpec
clientSetObjects []runtime.Object
createVolumeError error
expectErr bool
expectState controller.ProvisioningState
expectCreateVolDo func(t *testing.T, ctx context.Context, req *csi.CreateVolumeRequest)
withExtraMetadata bool
skipCreateVolume bool
deploymentNode string // fake distributed provisioning with this node as host
immediateBinding bool // enable immediate binding support for distributed provisioning
expectSelectedNode string // a specific selected-node of the PVC in the apiserver after the test, same as before if empty
expectNoProvision bool // if true, then ShouldProvision should return false
capacity int64 // if zero, default capacity, otherwise available bytes
volOpts controller.ProvisionOptions
notNilSelector bool
makeVolumeNameErr bool
getSecretRefErr bool
getCredentialsErr bool
volWithLessCap bool
volWithZeroCap bool
expectedPVSpec *pvSpec
clientSetObjects []runtime.Object
createVolumeError error
expectErr bool
expectState controller.ProvisioningState
expectCreateVolDo func(t *testing.T, ctx context.Context, req *csi.CreateVolumeRequest)
withExtraMetadata bool
skipCreateVolume bool
deploymentNode string // fake distributed provisioning with this node as host
immediateBinding bool // enable immediate binding support for distributed provisioning
expectSelectedNode string // a specific selected-node of the PVC in the apiserver after the test, same as before if empty
expectNoProvision bool // if true, then ShouldProvision should return false
supportsSingleNodeMultiWriter bool // if true, then provision with single node multi writer capabilities
}

type provisioningFSTypeTestcase struct {
Expand Down Expand Up @@ -1258,7 +1268,7 @@ func provisionTestcases() (int64, map[string]provisioningTestcase) {
},
expectState: controller.ProvisioningFinished,
},
"provision with access mode single writer": {
"provision with access mode single node writer": {
volOpts: controller.ProvisionOptions{
StorageClass: &storagev1.StorageClass{
ReclaimPolicy: &deletePolicy,
Expand Down Expand Up @@ -1310,6 +1320,112 @@ func provisionTestcases() (int64, map[string]provisioningTestcase) {
},
expectState: controller.ProvisioningFinished,
},
"provision with access mode single node writer and single node multi writer capability": {
supportsSingleNodeMultiWriter: true,
volOpts: controller.ProvisionOptions{
StorageClass: &storagev1.StorageClass{
ReclaimPolicy: &deletePolicy,
Parameters: map[string]string{},
},
PVName: "test-name",
PVC: &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
UID: "testid",
Annotations: driverNameAnnotation,
},
Spec: v1.PersistentVolumeClaimSpec{
Selector: nil,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse(strconv.FormatInt(requestedBytes, 10)),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
},
},
},
expectedPVSpec: &pvSpec{
Name: "test-testi",
ReclaimPolicy: v1.PersistentVolumeReclaimDelete,
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): bytesToQuantity(requestedBytes),
},
CSIPVS: &v1.CSIPersistentVolumeSource{
Driver: "test-driver",
VolumeHandle: "test-volume-id",
FSType: "ext4",
VolumeAttributes: map[string]string{
"storage.kubernetes.io/csiProvisionerIdentity": "test-provisioner",
},
},
},
expectCreateVolDo: func(t *testing.T, ctx context.Context, req *csi.CreateVolumeRequest) {
if len(req.GetVolumeCapabilities()) != 1 {
t.Errorf("Incorrect length in volume capabilities")
}
if req.GetVolumeCapabilities()[0].GetAccessMode() == nil {
t.Errorf("Expected access mode to be set")
}
if req.GetVolumeCapabilities()[0].GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER {
t.Errorf("Expected single_node_multi_writer")
}
},
expectState: controller.ProvisioningFinished,
},
"provision with access mode single node single writer and single node multi writer capability": {
supportsSingleNodeMultiWriter: true,
volOpts: controller.ProvisionOptions{
StorageClass: &storagev1.StorageClass{
ReclaimPolicy: &deletePolicy,
Parameters: map[string]string{},
},
PVName: "test-name",
PVC: &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
UID: "testid",
Annotations: driverNameAnnotation,
},
Spec: v1.PersistentVolumeClaimSpec{
Selector: nil,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse(strconv.FormatInt(requestedBytes, 10)),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod},
},
},
},
expectedPVSpec: &pvSpec{
Name: "test-testi",
ReclaimPolicy: v1.PersistentVolumeReclaimDelete,
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod},
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): bytesToQuantity(requestedBytes),
},
CSIPVS: &v1.CSIPersistentVolumeSource{
Driver: "test-driver",
VolumeHandle: "test-volume-id",
FSType: "ext4",
VolumeAttributes: map[string]string{
"storage.kubernetes.io/csiProvisionerIdentity": "test-provisioner",
},
},
},
expectCreateVolDo: func(t *testing.T, ctx context.Context, req *csi.CreateVolumeRequest) {
if len(req.GetVolumeCapabilities()) != 1 {
t.Errorf("Incorrect length in volume capabilities")
}
if req.GetVolumeCapabilities()[0].GetAccessMode() == nil {
t.Errorf("Expected access mode to be set")
}
if req.GetVolumeCapabilities()[0].GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER {
t.Errorf("Expected single_node_multi_writer")
}
},
expectState: controller.ProvisioningFinished,
},
"provision with multiple access modes": {
volOpts: controller.ProvisionOptions{
StorageClass: &storagev1.StorageClass{
Expand Down Expand Up @@ -2299,7 +2415,14 @@ func runProvisionTest(t *testing.T, tc provisioningTestcase, requestedBytes int6
}
}

pluginCaps, controllerCaps := provisionCapabilities()
var pluginCaps rpc.PluginCapabilitySet
var controllerCaps rpc.ControllerCapabilitySet
if tc.supportsSingleNodeMultiWriter {
pluginCaps, controllerCaps = provisionWithSingleNodeMultiWriterCapabilities()
} else {
pluginCaps, controllerCaps = provisionCapabilities()
}

csiProvisioner := NewCSIProvisioner(clientSet, 5*time.Second, "test-provisioner", "test", 5, csiConn.conn,
nil, provisionDriverName, pluginCaps, controllerCaps, supportsMigrationFromInTreePluginName, false, true, csitrans.New(), scInformer.Lister(), csiNodeInformer.Lister(), nodeInformer.Lister(), nil, nil, tc.withExtraMetadata, defaultfsType, nodeDeployment)

Expand Down

0 comments on commit 421bd42

Please sign in to comment.