provisioner/shell: start_retry_timeout for reboot handling [GH-260]

CHANGELOG.md

@@ -1,6 +1,10 @@
 ## 0.3.1 (unreleased)
 
+IMPROVEMENTS:
 
+* provisioner/shell: New setting `start_retry_timeout` which is the timeout
+  for the provisioner to attempt to _start_ the remote process. This allows
+  the shell provisioner to work properly with reboots. [GH-260]
 
 ## 0.3.0 (August 12, 2013)
 

provisioner/shell/provisioner.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 )
 
 const DefaultRemotePath = "/tmp/script.sh"
@@ -45,7 +46,13 @@ type config struct {
 	// can be used to inject the environment_vars into the environment.
 	ExecuteCommand string `mapstructure:"execute_command"`
 
-	tpl *common.Template
+	// The timeout for retrying to start the process. Until this timeout
+	// is reached, if the provisioner can't start a process, it retries.
+	// This can be set high to allow for reboots.
+	RawStartRetryTimeout string `mapstructure:"start_retry_timeout"`
+
+	startRetryTimeout time.Duration
+	tpl               *common.Template
 }
 
 type Provisioner struct {
@@ -84,6 +91,10 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 		p.config.InlineShebang = "/bin/sh"
 	}
 
+	if p.config.RawStartRetryTimeout == "" {
+		p.config.RawStartRetryTimeout = "5m"
+	}
+
 	if p.config.RemotePath == "" {
 		p.config.RemotePath = DefaultRemotePath
 	}
@@ -106,9 +117,10 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 	}
 
 	templates := map[string]*string{
-		"inline_shebang": &p.config.InlineShebang,
-		"script":         &p.config.Script,
-		"remote_path":    &p.config.RemotePath,
+		"inline_shebang":      &p.config.InlineShebang,
+		"script":              &p.config.Script,
+		"start_retry_timeout": &p.config.RawStartRetryTimeout,
+		"remote_path":         &p.config.RemotePath,
 	}
 
 	for n, ptr := range templates {
@@ -161,6 +173,14 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 		}
 	}
 
+	if p.config.RawStartRetryTimeout != "" {
+		p.config.startRetryTimeout, err = time.ParseDuration(p.config.RawStartRetryTimeout)
+		if err != nil {
+			errs = packer.MultiErrorAppend(
+				errs, fmt.Errorf("Failed parsing start_retry_timeout: %s", err))
+		}
+	}
+
 	if errs != nil && len(errs.Errors) > 0 {
 		return errs
 	}
@@ -238,9 +258,26 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 		}
 
 		cmd := &packer.RemoteCmd{Command: command}
+		startTimeout := time.After(p.config.startRetryTimeout)
 		log.Printf("Executing command: %s", cmd.Command)
-		if err := cmd.StartWithUi(comm, ui); err != nil {
-			return fmt.Errorf("Failed executing command: %s", err)
+		for {
+			if err := cmd.StartWithUi(comm, ui); err == nil {
+				break
+			}
+
+			// Create an error and log it
+			err = fmt.Errorf("Error executing command: %s", err)
+			log.Printf(err.Error())
+
+			// Check if we timed out, otherwise we retry. It is safe to
+			// retry since the only error case above is if the command
+			// failed to START.
+			select {
+			case <-startTimeout:
+				return err
+			default:
+				time.Sleep(2 * time.Second)
+			}
 		}
 
 		if cmd.ExitStatus != 0 {

website/source/docs/provisioners/shell.html.markdown

@@ -67,6 +67,12 @@ Optional parameters:
   in the machine. This defaults to "/tmp/script.sh". This value must be
   a writable location and any parent directories must already exist.
 
+* `start_retry_timeout` (string) - The amount of time to attempt to
+  _start_ the remote process. By default this is "5m" or 5 minutes. This
+  setting exists in order to deal with times when SSH may restart, such as
+  a system reboot. Set this to a higher value if reboots take a longer
+  amount of time.
+
 ## Execute Command Example
 
 To many new users, the `execute_command` is puzzling. However, it provides