Support GPU instances, allow set source image by name

Signed-off-by: Gennady Lipenkov <xgen@yandex-team.ru>

builder/yandex/builder.go

@@ -61,7 +61,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
 		&stepInstanceInfo{},
 		&communicator.StepConnect{
 			Config:    &b.config.Communicator,
-			Host:      communicator.CommHost(b.config.Communicator.SSHHost, "instance_ip"),
+			Host:      commHost,
 			SSHConfig: b.config.Communicator.SSHConfigFunc(),
 		},
 		&common.StepProvision{},

builder/yandex/config.go

@@ -18,6 +18,8 @@ import (
 )
 
 const defaultEndpoint = "api.cloud.yandex.net:443"
+const defaultGpuPlatformID = "gpu-standard-v1"
+const defaultPlatformID = "standard-v1"
 const defaultZone = "ru-central1-a"
 
 var reImageFamily = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`)
@@ -40,6 +42,7 @@ type Config struct {
 	ImageName           string            `mapstructure:"image_name"`
 	ImageProductIDs     []string          `mapstructure:"image_product_ids"`
 	InstanceCores       int               `mapstructure:"instance_cores"`
+	InstanceGpus        int               `mapstructure:"instance_gpus"`
 	InstanceMemory      int               `mapstructure:"instance_mem_gb"`
 	InstanceName        string            `mapstructure:"instance_name"`
 	Labels              map[string]string `mapstructure:"labels"`
@@ -51,6 +54,7 @@ type Config struct {
 	SourceImageFamily   string            `mapstructure:"source_image_family"`
 	SourceImageFolderID string            `mapstructure:"source_image_folder_id"`
 	SourceImageID       string            `mapstructure:"source_image_id"`
+	SourceImageName     string            `mapstructure:"source_image_name"`
 	SubnetID            string            `mapstructure:"subnet_id"`
 	UseIPv4Nat          bool              `mapstructure:"use_ipv4_nat"`
 	UseIPv6             bool              `mapstructure:"use_ipv6"`
@@ -134,7 +138,10 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 	}
 
 	if c.PlatformID == "" {
-		c.PlatformID = "standard-v1"
+		c.PlatformID = defaultPlatformID
+		if c.InstanceGpus != 0 {
+			c.PlatformID = defaultGpuPlatformID
+		}
 	}
 
 	if es := c.Communicator.Prepare(&c.ctx); len(es) > 0 {
@@ -142,9 +149,15 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 	}
 
 	// Process required parameters.
-	if c.SourceImageID == "" && c.SourceImageFamily == "" {
-		errs = packer.MultiErrorAppend(
-			errs, errors.New("a source_image_id or source_image_family must be specified"))
+	if c.SourceImageID == "" {
+		if c.SourceImageFamily == "" && c.SourceImageName == "" {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("a source_image_name or source_image_family must be specified"))
+		}
+		if c.SourceImageFamily != "" && c.SourceImageName != "" {
+			errs = packer.MultiErrorAppend(
+				errs, errors.New("one of source_image_name or source_image_family must be specified, not both"))
+		}
 	}
 
 	if c.Endpoint == "" {
@@ -168,6 +181,11 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 		c.FolderID = os.Getenv("YC_FOLDER_ID")
 	}
 
+	if c.PlatformID != defaultGpuPlatformID && c.InstanceGpus != 0 {
+		errs = packer.MultiErrorAppend(
+			errs, fmt.Errorf("for instances with gpu platform_id must be specified as `%s`", defaultGpuPlatformID))
+	}
+
 	if c.Token == "" && c.ServiceAccountKeyFile == "" {
 		errs = packer.MultiErrorAppend(
 			errs, errors.New("a token or service account key file must be specified"))

builder/yandex/config_test.go

@@ -214,6 +214,25 @@ func TestZone(t *testing.T) {
 	}
 }
 
+func TestGpuDefaultPlatformID(t *testing.T) {
+	raw := testConfig(t)
+	raw["instance_gpus"] = 1
+
+	c, _, _ := NewConfig(raw)
+	if c.PlatformID != "gpu-standard-v1" {
+		t.Fatalf("expected 'gpu-standard-v1', but got '%s'", c.PlatformID)
+	}
+}
+
+func TestGpuWrongPlatformID(t *testing.T) {
+	raw := testConfig(t)
+	raw["instance_gpus"] = 1
+	raw["platform_id"] = "standard-v1"
+
+	_, warns, errs := NewConfig(raw)
+	testConfigErr(t, warns, errs, "incompatible GPU platform_id")
+}
+
 // Helper stuff below
 
 func testConfig(t *testing.T) (config map[string]interface{}) {

builder/yandex/driver.go

@@ -11,6 +11,7 @@ type Driver interface {
 	SDK() *ycsdk.SDK
 	GetImage(imageID string) (*Image, error)
 	GetImageFromFolder(ctx context.Context, folderID string, family string) (*Image, error)
+	GetImageFromFolderByName(ctx context.Context, folderID string, name string) (*Image, error)
 	DeleteDisk(ctx context.Context, diskID string) error
 	DeleteInstance(ctx context.Context, instanceID string) error
 	DeleteSubnet(ctx context.Context, subnetID string) error

builder/yandex/driver_yc.go

@@ -2,6 +2,7 @@ package yandex
 
 import (
 	"context"
+	"fmt"
 	"log"
 
 	"github.com/hashicorp/packer/helper/useragent"
@@ -15,6 +16,7 @@ import (
 	ycsdk "github.com/yandex-cloud/go-sdk"
 	"github.com/yandex-cloud/go-sdk/iamkey"
 	"github.com/yandex-cloud/go-sdk/pkg/requestid"
+	"github.com/yandex-cloud/go-sdk/sdkresolvers"
 )
 
 type driverYC struct {
@@ -110,6 +112,16 @@ func (d *driverYC) GetImageFromFolder(ctx context.Context, folderID string, fami
 	}, nil
 }
 
+func (d *driverYC) GetImageFromFolderByName(ctx context.Context, folderID string, imageName string) (*Image, error) {
+	imageResolver := sdkresolvers.ImageResolver(imageName, sdkresolvers.FolderID(folderID))
+
+	if err := d.sdk.Resolve(ctx, imageResolver); err != nil {
+		return nil, fmt.Errorf("failed to resolve image name: %s", err)
+	}
+
+	return d.GetImage(imageResolver.ID())
+}
+
 func (d *driverYC) DeleteImage(ID string) error {
 	ctx := context.TODO()
 	op, err := d.sdk.WrapOperation(d.sdk.Compute().Image().Delete(ctx, &compute.DeleteImageRequest{

builder/yandex/ssh.go

@@ -0,0 +1,10 @@
+package yandex
+
+import (
+	"github.com/hashicorp/packer/helper/multistep"
+)
+
+func commHost(state multistep.StateBag) (string, error) {
+	ipAddress := state.Get("instance_ip").(string)
+	return ipAddress, nil
+}

builder/yandex/step_create_instance.go

@@ -91,11 +91,19 @@ func getImage(ctx context.Context, c *Config, d Driver) (*Image, error) {
 		return d.GetImage(c.SourceImageID)
 	}
 
-	familyName := c.SourceImageFamily
-	if c.SourceImageFolderID != "" {
-		return d.GetImageFromFolder(ctx, c.SourceImageFolderID, familyName)
+	folderID := c.SourceImageFolderID
+	if folderID == "" {
+		folderID = StandardImagesFolderID
 	}
-	return d.GetImageFromFolder(ctx, StandardImagesFolderID, familyName)
+
+	switch {
+	case c.SourceImageFamily != "":
+		return d.GetImageFromFolder(ctx, folderID, c.SourceImageFamily)
+	case c.SourceImageName != "":
+		return d.GetImageFromFolderByName(ctx, folderID, c.SourceImageName)
+	}
+
+	return &Image{}, errors.New("neither source_image_name nor source_image_family defined in config")
 }
 
 func (s *stepCreateInstance) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
@@ -173,6 +181,7 @@ runcmd:
 		ResourcesSpec: &compute.ResourcesSpec{
 			Memory: toBytes(config.InstanceMemory),
 			Cores:  int64(config.InstanceCores),
+			Gpus:   int64(config.InstanceGpus),
 		},
 		Metadata: instanceMetadata,
 		BootDiskSpec: &compute.AttachedDiskSpec{

builder/yandex/step_instance_info.go

@@ -20,7 +20,7 @@ func (s *stepInstanceInfo) Run(ctx context.Context, state multistep.StateBag) mu
 	c := state.Get("config").(*Config)
 	instanceID := state.Get("instance_id").(string)
 
-	ui.Say("Waiting for instance to become active...")
+	ui.Say(fmt.Sprintf("Waiting for instance with id %s to become active...", instanceID))
 
 	ctx, cancel := context.WithTimeout(ctx, c.StateTimeout)
 	defer cancel()

website/source/docs/builders/yandex.html.md

@@ -72,15 +72,21 @@ can be configured for this builder.
 
 ### Optional:
 
+-   `endpoint` (string) - Non standard api endpoint URL.
+
+-   `instance_cores` (number) - The number of cores available to the instance. 
+
+-   `instance_gpus` (number) - The number of GPU available to the instance. 
+
+-   `instance_mem_gb` (number) - The amount of memory available to the instance, specified in gigabytes.
+
 -   `disk_name` (string) - The name of the disk, if unset the instance name
     will be used.
-
+    
 -   `disk_size_gb` (number) - The size of the disk in GB. This defaults to `10`, which is 10GB.
 
 -   `disk_type` (string) - Specify disk type for the launched instance. Defaults to `network-hdd`.
 
--   `endpoint` (string) - Non standard api endpoint URL.
-
 -   `image_description` (string) - The description of the resulting image.
 
 -   `image_family` (string) -  The family name of the resulting image.
@@ -93,21 +99,15 @@ can be configured for this builder.
 
 -   `image_product_ids` (list) - License IDs that indicate which licenses are attached to resulting image.
 
--   `instance_cores` (number) - The number of cores available to the instance. 
-
--   `instance_mem_gb` (number) - The amount of memory available to the instance, specified in gigabytes.    
-
 -   `instance_name` (string) - The name assigned to the instance.
                 
 -   `labels` (object of key/value strings) - Key/value pair labels to apply to
     the launched instance.
     
--   `metadata` (object of key/value strings) - Metadata applied to the launched
-    instance.
-
 -   `platform_id` (string) - Identifier of the hardware platform configuration for the instance. This defaults to `standard-v1`.
 
--   `preemptible` (boolean) - Launch a preemptible instance. This defaults to `false`.
+-   `metadata` (object of key/value strings) - Metadata applied to the launched
+    instance.
 
 -   `serial_log_file` (string) - File path to save serial port output of the launched instance.
 
@@ -115,18 +115,10 @@ can be configured for this builder.
     is an alternative method to authenticate to Yandex.Cloud. Alternatively you may set environment variable
     `YC_SERVICE_ACCOUNT_KEY_FILE`.
 
--   `source_image_folder_id` (string) - The ID of the folder containing the source image.
-
--   `source_image_id` (string) - The source image ID to use to create the new image
-    from.
-
 -   `source_image_family` (string) - The source image family to create
     the new image from. The image family always returns its latest image that
     is not deprecated. Example: `ubuntu-1804-lts`.
 
--   `state_timeout` (string) - The time to wait for instance state changes.
-    Defaults to `5m`.
-
 -   `subnet_id` (string) - The Yandex VPC subnet id to use for 
     the launched instance. Note, the zone of the subnet must match the
     `zone` in which the VM is launched. 
@@ -141,4 +133,14 @@ can be configured for this builder.
     created. This defaults to `false`, or not enabled.
 -> **Note:** ~> Usage of IPv6 will be available in the future.
 
+-   `source_image_folder_id` (string) - The ID of the folder containing the source image.
+
+-   `source_image_id` (string) - The source image ID to use to create the new image from.
+
+-   `source_image_name` (string) - The source image name to use to create the new image from. 
+    Name will be looked up in `source_image_folder_id`. 
+
+-   `state_timeout` (string) - The time to wait for instance state changes.
+    Defaults to `5m`.
+
 -   `zone` (string) - The name of the zone to launch the instance.  This defaults to `ru-central1-a`.