Skip to content

Commit

Permalink
Add local_ssd_interface to Dataproc Cluster's disk_config. (#1066…
Browse files Browse the repository at this point in the history 
…3) (#7366)

[upstream:87e753c0b86b3e5f00938d90c2080bf2748d62b8]

Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
@modular-magician
modular-magician committed May 14, 2024
1 parent a2226ac commit 1163ddf
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 63 deletions.
159 changes: 96 additions & 63 deletions google-beta/services/dataproc/resource_dataproc_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,9 @@ var (
"cluster_config.0.gce_cluster_config.0.reservation_affinity.0.values",
}

preemptibleWorkerDiskConfigKeys = []string{
"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type",
}
masterDiskConfigKeys = diskConfigKeys("master_config")
workerDiskConfigKeys = diskConfigKeys("worker_config")
preemptibleWorkerDiskConfigKeys = diskConfigKeys("preemptible_worker_config")

clusterSoftwareConfigKeys = []string{
"cluster_config.0.software_config.0.image_version",
Expand Down Expand Up @@ -122,6 +120,18 @@ var (
const resourceDataprocGoogleLabelPrefix = "goog-dataproc"
const resourceDataprocGoogleProvidedLabelPrefix = "labels." + resourceDataprocGoogleLabelPrefix

// The keys inside a DiskConfig. configName is the name of the field this disk
// config is inside. E.g. 'master_config', 'worker_config', or
// 'preemptible_worker_config'.
func diskConfigKeys(configName string) []string {
return []string{
"cluster_config.0." + configName + ".0.disk_config.0.num_local_ssds",
"cluster_config.0." + configName + ".0.disk_config.0.boot_disk_size_gb",
"cluster_config.0." + configName + ".0.disk_config.0.boot_disk_type",
"cluster_config.0." + configName + ".0.disk_config.0.local_ssd_interface",
}
}

func resourceDataprocLabelDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
if strings.HasPrefix(k, resourceDataprocGoogleProvidedLabelPrefix) && new == "" {
return true
Expand Down Expand Up @@ -821,43 +831,39 @@ func ResourceDataprocCluster() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_local_ssds": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
AtLeastOneOf: []string{
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
},
ForceNew: true,
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
AtLeastOneOf: masterDiskConfigKeys,
ForceNew: true,
},

"boot_disk_size_gb": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
AtLeastOneOf: []string{
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
},
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
AtLeastOneOf: masterDiskConfigKeys,
ForceNew: true,
ValidateFunc: validation.IntAtLeast(10),
},

"boot_disk_type": {
Type: schema.TypeString,
Optional: true,
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
AtLeastOneOf: []string{
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
},
ForceNew: true,
Default: "pd-standard",
Type: schema.TypeString,
Optional: true,
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
AtLeastOneOf: masterDiskConfigKeys,
ForceNew: true,
Default: "pd-standard",
},

"local_ssd_interface": {
Type: schema.TypeString,
Optional: true,
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
AtLeastOneOf: masterDiskConfigKeys,
ForceNew: true,
},
},
},
Expand Down Expand Up @@ -970,43 +976,39 @@ func ResourceDataprocCluster() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_local_ssds": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
AtLeastOneOf: []string{
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
},
ForceNew: true,
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
AtLeastOneOf: workerDiskConfigKeys,
ForceNew: true,
},

"boot_disk_size_gb": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
AtLeastOneOf: []string{
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
},
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
AtLeastOneOf: workerDiskConfigKeys,
ForceNew: true,
ValidateFunc: validation.IntAtLeast(10),
},

"boot_disk_type": {
Type: schema.TypeString,
Optional: true,
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
AtLeastOneOf: []string{
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
},
ForceNew: true,
Default: "pd-standard",
Type: schema.TypeString,
Optional: true,
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
AtLeastOneOf: workerDiskConfigKeys,
ForceNew: true,
Default: "pd-standard",
},

"local_ssd_interface": {
Type: schema.TypeString,
Optional: true,
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
AtLeastOneOf: workerDiskConfigKeys,
ForceNew: true,
},
},
},
Expand Down Expand Up @@ -1132,6 +1134,14 @@ func ResourceDataprocCluster() *schema.Resource {
Default: "pd-standard",
Description: `The disk type of the primary disk attached to each preemptible worker node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
},

"local_ssd_interface": {
Type: schema.TypeString,
Optional: true,
AtLeastOneOf: preemptibleWorkerDiskConfigKeys,
ForceNew: true,
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
},
},
},
},
Expand Down Expand Up @@ -1616,6 +1626,13 @@ by Dataproc`,
ForceNew: true,
Default: "pd-standard",
},

"local_ssd_interface": {
Type: schema.TypeString,
Optional: true,
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
ForceNew: true,
},
},
},
},
Expand Down Expand Up @@ -2096,6 +2113,9 @@ func expandNodeGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGroupCo
if v, ok := dcfg["boot_disk_type"]; ok {
icg.DiskConfig.BootDiskType = v.(string)
}
if v, ok := dcfg["local_ssd_interface"]; ok {
icg.DiskConfig.LocalSsdInterface = v.(string)
}
}
}

Expand Down Expand Up @@ -2368,6 +2388,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
if v, ok := dcfg["boot_disk_type"]; ok {
icg.DiskConfig.BootDiskType = v.(string)
}
if v, ok := dcfg["local_ssd_interface"]; ok {
icg.DiskConfig.LocalSsdInterface = v.(string)
}
}
}

Expand Down Expand Up @@ -2441,6 +2464,9 @@ func expandMasterInstanceGroupConfig(cfg map[string]interface{}) *dataproc.Insta
if v, ok := dcfg["boot_disk_type"]; ok {
icg.DiskConfig.BootDiskType = v.(string)
}
if v, ok := dcfg["local_ssd_interface"]; ok {
icg.DiskConfig.LocalSsdInterface = v.(string)
}
}
}

Expand Down Expand Up @@ -2482,6 +2508,9 @@ func expandWorkerInstanceGroupConfig(cfg map[string]interface{}) *dataproc.Insta
if v, ok := dcfg["boot_disk_type"]; ok {
icg.DiskConfig.BootDiskType = v.(string)
}
if v, ok := dcfg["local_ssd_interface"]; ok {
icg.DiskConfig.LocalSsdInterface = v.(string)
}
}
}

Expand Down Expand Up @@ -3048,6 +3077,7 @@ func flattenNodeGroupConfig(icg *dataproc.InstanceGroupConfig) []map[string]inte
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
}
data["accelerators"] = flattenAccelerators(icg.Accelerators)

Expand Down Expand Up @@ -3137,6 +3167,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
}
if icg.InstanceFlexibilityPolicy != nil {
instanceFlexibilityPolicy["instance_selection_list"] = flattenInstanceSelectionList(icg.InstanceFlexibilityPolicy.InstanceSelectionList)
Expand Down Expand Up @@ -3191,6 +3222,7 @@ func flattenMasterInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.Inst
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
}

data["accelerators"] = flattenAccelerators(icg.Accelerators)
Expand All @@ -3215,6 +3247,7 @@ func flattenWorkerInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.Inst
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
}

data["accelerators"] = flattenAccelerators(icg.Accelerators)
Expand Down
8 changes: 8 additions & 0 deletions google-beta/services/dataproc/resource_dataproc_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,7 @@ func TestAccDataprocCluster_spotWithAuxiliaryNodeGroups(t *testing.T) {
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.boot_disk_size_gb", "35"),
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.boot_disk_type", "pd-standard"),
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.num_local_ssds", "1"),
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.local_ssd_interface", "nvme"),
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.accelerators.0.accelerator_count", "1"),
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group_id", "node-group-id"),
testAccCheckDataprocAuxiliaryNodeGroupAccelerator(&cluster, project),
Expand Down Expand Up @@ -1227,6 +1228,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
{"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.BootDiskSizeGb))},
{"cluster_config.0.master_config.0.disk_config.0.num_local_ssds", "0", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.NumLocalSsds))},
{"cluster_config.0.master_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.MasterConfig.DiskConfig.BootDiskType},
{"cluster_config.0.master_config.0.disk_config.0.local_ssd_interface", "nvme", cluster.Config.MasterConfig.DiskConfig.LocalSsdInterface},
{"cluster_config.0.master_config.0.machine_type", "n1-standard-2", tpgresource.GetResourceNameFromSelfLink(cluster.Config.MasterConfig.MachineTypeUri)},
{"cluster_config.0.master_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.MasterConfig.InstanceNames))},
{"cluster_config.0.master_config.0.min_cpu_platform", "Intel Skylake", cluster.Config.MasterConfig.MinCpuPlatform},
Expand All @@ -1235,6 +1237,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
{"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.BootDiskSizeGb))},
{"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.NumLocalSsds))},
{"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type", "pd-standard", cluster.Config.WorkerConfig.DiskConfig.BootDiskType},
{"cluster_config.0.worker_config.0.disk_config.0.local_ssd_interface", "scsi", cluster.Config.WorkerConfig.DiskConfig.LocalSsdInterface},
{"cluster_config.0.worker_config.0.machine_type", "n1-standard-2", tpgresource.GetResourceNameFromSelfLink(cluster.Config.WorkerConfig.MachineTypeUri)},
{"cluster_config.0.worker_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.WorkerConfig.InstanceNames))},
{"cluster_config.0.worker_config.0.min_cpu_platform", "Intel Broadwell", cluster.Config.WorkerConfig.MinCpuPlatform},
Expand All @@ -1243,6 +1246,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskSizeGb))},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.NumLocalSsds))},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskType},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.local_ssd_interface", "nvme", cluster.Config.SecondaryWorkerConfig.DiskConfig.LocalSsdInterface},
{"cluster_config.0.preemptible_worker_config.0.instance_names.#", "1", strconv.Itoa(len(cluster.Config.SecondaryWorkerConfig.InstanceNames))},
}

Expand Down Expand Up @@ -1695,6 +1699,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
disk_config {
boot_disk_type = "pd-ssd"
boot_disk_size_gb = 35
local_ssd_interface = "nvme"
}
min_cpu_platform = "Intel Skylake"
}
Expand All @@ -1706,6 +1711,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
boot_disk_type = "pd-standard"
boot_disk_size_gb = 35
num_local_ssds = 1
local_ssd_interface = "scsi"
}
min_cpu_platform = "Intel Broadwell"
Expand All @@ -1717,6 +1723,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
boot_disk_type = "pd-ssd"
boot_disk_size_gb = 35
num_local_ssds = 1
local_ssd_interface = "nvme"
}
}
}
Expand Down Expand Up @@ -1967,6 +1974,7 @@ resource "google_dataproc_cluster" "with_auxiliary_node_groups" {
boot_disk_size_gb = 35
boot_disk_type = "pd-standard"
num_local_ssds = 1
local_ssd_interface = "nvme"
}
accelerators {
accelerator_count = 1
Expand Down
5 changes: 5 additions & 0 deletions website/docs/r/dataproc_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,11 @@ cluster_config {
* `num_local_ssds` - (Optional) The amount of local SSD disks that will be
attached to each master cluster node. Defaults to 0.

* `local_ssd_interface` - Optional. Interface type of local SSDs (default is "scsi").
Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile
Memory Express). See
[local SSD performance](https://cloud.google.com/compute/docs/disks/local-ssd#performance).

* `accelerators` (Optional) The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times.

* `accelerator_type` - (Required) The short name of the accelerator type to expose to this instance. For example, `nvidia-tesla-k80`.
Expand Down

0 comments on commit 1163ddf

Please sign in to comment.