Merge pull request #11411 from hashicorp/f-emr-instance-ebs-volumes

provider/aws: Add EBS Volume support for EMR Instance Groups

builtin/providers/aws/resource_aws_emr_instance_group.go

@@ -22,38 +22,100 @@ func resourceAwsEMRInstanceGroup() *schema.Resource {
 		Update: resourceAwsEMRInstanceGroupUpdate,
 		Delete: resourceAwsEMRInstanceGroupDelete,
 		Schema: map[string]*schema.Schema{
-			"cluster_id": &schema.Schema{
+			"cluster_id": {
 				Type:     schema.TypeString,
 				Required: true,
 				ForceNew: true,
 			},
-			"instance_type": &schema.Schema{
+			"instance_type": {
 				Type:     schema.TypeString,
 				Required: true,
 				ForceNew: true,
 			},
-			"instance_count": &schema.Schema{
+			"instance_count": {
 				Type:     schema.TypeInt,
 				Optional: true,
 				Default:  0,
 			},
-			"running_instance_count": &schema.Schema{
+			"running_instance_count": {
 				Type:     schema.TypeInt,
 				Computed: true,
 			},
-			"status": &schema.Schema{
+			"status": {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
-			"name": &schema.Schema{
+			"name": {
 				Type:     schema.TypeString,
 				Optional: true,
 				ForceNew: true,
 			},
+			"ebs_optimized": {
+				Type:     schema.TypeBool,
+				Optional: true,
+				ForceNew: true,
+			},
+			"ebs_config": {
+				Type:     schema.TypeSet,
+				Optional: true,
+				ForceNew: true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"iops": {
+							Type:     schema.TypeInt,
+							Optional: true,
+						},
+						"size": {
+							Type:     schema.TypeInt,
+							Required: true,
+						},
+						"type": {
+							Type:         schema.TypeString,
+							Required:     true,
+							ValidateFunc: validateAwsEmrEbsVolumeType,
+						},
+						"volumes_per_instance": {
+							Type:     schema.TypeInt,
+							Optional: true,
+						},
+					},
+				},
+			},
 		},
 	}
 }
 
+// Populates an emr.EbsConfiguration struct
+func readEmrEBSConfig(d *schema.ResourceData) *emr.EbsConfiguration {
+	result := &emr.EbsConfiguration{}
+	if v, ok := d.GetOk("ebs_optimized"); ok {
+		result.EbsOptimized = aws.Bool(v.(bool))
+	}
+
+	ebsConfigs := make([]*emr.EbsBlockDeviceConfig, 0)
+	if rawConfig, ok := d.GetOk("ebs_config"); ok {
+		configList := rawConfig.(*schema.Set).List()
+		for _, config := range configList {
+			conf := config.(map[string]interface{})
+			ebs := &emr.EbsBlockDeviceConfig{}
+			volumeSpec := &emr.VolumeSpecification{
+				SizeInGB:   aws.Int64(int64(conf["size"].(int))),
+				VolumeType: aws.String(conf["type"].(string)),
+			}
+			if v, ok := conf["iops"].(int); ok && v != 0 {
+				volumeSpec.Iops = aws.Int64(int64(v))
+			}
+			if v, ok := conf["volumes_per_instance"].(int); ok && v != 0 {
+				ebs.VolumesPerInstance = aws.Int64(int64(v))
+			}
+			ebs.VolumeSpecification = volumeSpec
+			ebsConfigs = append(ebsConfigs, ebs)
+		}
+	}
+	result.EbsBlockDeviceConfigs = ebsConfigs
+	return result
+}
+
 func resourceAwsEMRInstanceGroupCreate(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*AWSClient).emrconn
 
@@ -62,13 +124,16 @@ func resourceAwsEMRInstanceGroupCreate(d *schema.ResourceData, meta interface{})
 	instanceCount := d.Get("instance_count").(int)
 	groupName := d.Get("name").(string)
 
+	ebsConfig := readEmrEBSConfig(d)
+
 	params := &emr.AddInstanceGroupsInput{
 		InstanceGroups: []*emr.InstanceGroupConfig{
 			{
-				InstanceRole:  aws.String("TASK"),
-				InstanceCount: aws.Int64(int64(instanceCount)),
-				InstanceType:  aws.String(instanceType),
-				Name:          aws.String(groupName),
+				InstanceRole:     aws.String("TASK"),
+				InstanceCount:    aws.Int64(int64(instanceCount)),
+				InstanceType:     aws.String(instanceType),
+				Name:             aws.String(groupName),
+				EbsConfiguration: ebsConfig,
 			},
 		},
 		JobFlowId: aws.String(clusterId),

builtin/providers/aws/resource_aws_emr_instance_group_test.go

@@ -21,7 +21,7 @@ func TestAccAWSEMRInstanceGroup_basic(t *testing.T) {
 		Providers:    testAccProviders,
 		CheckDestroy: testAccCheckAWSEmrInstanceGroupDestroy,
 		Steps: []resource.TestStep{
-			resource.TestStep{
+			{
 				Config: testAccAWSEmrInstanceGroupConfig(rInt),
 				Check:  testAccCheckAWSEmrInstanceGroupExists("aws_emr_instance_group.task", &ig),
 			},
@@ -29,6 +29,28 @@ func TestAccAWSEMRInstanceGroup_basic(t *testing.T) {
 	})
 }
 
+func TestAccAWSEMRInstanceGroup_ebsBasic(t *testing.T) {
+	var ig emr.InstanceGroup
+	rInt := acctest.RandInt()
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckAWSEmrInstanceGroupDestroy,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccAWSEmrInstanceGroupConfig_ebsBasic(rInt),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckAWSEmrInstanceGroupExists("aws_emr_instance_group.task", &ig),
+					resource.TestCheckResourceAttr(
+						"aws_emr_instance_group.task", "ebs_config.#", "1"),
+					resource.TestCheckResourceAttr(
+						"aws_emr_instance_group.task", "ebs_optimized", "true"),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckAWSEmrInstanceGroupDestroy(s *terraform.State) error {
 	conn := testAccProvider.Meta().(*AWSClient).emrconn
 
@@ -85,8 +107,7 @@ func testAccCheckAWSEmrInstanceGroupExists(n string, v *emr.InstanceGroup) resou
 	}
 }
 
-func testAccAWSEmrInstanceGroupConfig(r int) string {
-	return fmt.Sprintf(`
+const testAccAWSEmrInstanceGroupBase = `
 provider "aws" {
   region = "us-west-2"
 }
@@ -126,12 +147,6 @@ resource "aws_emr_cluster" "tf-test-cluster" {
   depends_on = ["aws_internet_gateway.gw"]
 }
 
-resource "aws_emr_instance_group" "task" {
-  cluster_id     = "${aws_emr_cluster.tf-test-cluster.id}"
-  instance_count = 1
-  instance_type  = "m3.xlarge"
-}
-
 resource "aws_security_group" "allow_all" {
   name        = "allow_all"
   description = "Allow all inbound traffic"
@@ -352,5 +367,30 @@ resource "aws_iam_policy" "iam_emr_profile_policy" {
     }]
 }
 EOT
-}`, r, r, r, r, r, r)
+}
+`
+
+func testAccAWSEmrInstanceGroupConfig(r int) string {
+	return fmt.Sprintf(testAccAWSEmrInstanceGroupBase+`
+	resource "aws_emr_instance_group" "task" {
+    cluster_id     = "${aws_emr_cluster.tf-test-cluster.id}"
+    instance_count = 1
+    instance_type  = "m3.xlarge"
+  }
+	`, r, r, r, r, r, r)
+}
+
+func testAccAWSEmrInstanceGroupConfig_ebsBasic(r int) string {
+	return fmt.Sprintf(testAccAWSEmrInstanceGroupBase+`
+		resource "aws_emr_instance_group" "task" {
+    cluster_id     = "${aws_emr_cluster.tf-test-cluster.id}"
+    instance_count = 1
+    instance_type  = "m3.xlarge"
+    ebs_optimized = true
+    ebs_config {
+      "size" = 10,
+      "type" = "gp2",
+    }
+  }
+	`, r, r, r, r, r, r)
 }

builtin/providers/aws/validators.go

@@ -727,3 +727,19 @@ func validateAwsEcsPlacementStrategy(stratType, stratField string) error {
 	}
 	return nil
 }
+
+func validateAwsEmrEbsVolumeType(v interface{}, k string) (ws []string, errors []error) {
+	validTypes := map[string]struct{}{
+		"gp2":      {},
+		"io1":      {},
+		"standard": {},
+	}
+
+	value := v.(string)
+
+	if _, ok := validTypes[value]; !ok {
+		errors = append(errors, fmt.Errorf(
+			"%q must be one of ['gp2', 'io1', 'standard']", k))
+	}
+	return
+}

builtin/providers/aws/validators_test.go

@@ -1174,3 +1174,48 @@ func TestValidateEcsPlacementStrategy(t *testing.T) {
 
 	}
 }
+
+func TestValidateEmrEbsVolumeType(t *testing.T) {
+	cases := []struct {
+		VolType  string
+		ErrCount int
+	}{
+		{
+			VolType:  "gp2",
+			ErrCount: 0,
+		},
+		{
+			VolType:  "io1",
+			ErrCount: 0,
+		},
+		{
+			VolType:  "standard",
+			ErrCount: 0,
+		},
+		{
+			VolType:  "stand",
+			ErrCount: 1,
+		},
+		{
+			VolType:  "io",
+			ErrCount: 1,
+		},
+		{
+			VolType:  "gp1",
+			ErrCount: 1,
+		},
+		{
+			VolType:  "fast-disk",
+			ErrCount: 1,
+		},
+	}
+
+	for _, tc := range cases {
+		_, errors := validateAwsEmrEbsVolumeType(tc.VolType, "volume")
+
+		if len(errors) != tc.ErrCount {
+			t.Fatalf("Expected %d errors, got %d: %s", tc.ErrCount, len(errors), errors)
+		}
+	}
+
+}