diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go
index 302a90beb..cf644eb25 100644
--- a/builder/amazon/common/ssh.go
+++ b/builder/amazon/common/ssh.go
@@ -10,9 +10,9 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// SSHAddress returns a function that can be given to the SSH communicator
+// SSHHost returns a function that can be given to the SSH communicator
 // for determining the SSH address based on the instance DNS name.
-func SSHAddress(e *ec2.EC2, port int, private bool) func(multistep.StateBag) (string, error) {
+func SSHHost(e *ec2.EC2, private bool) func(multistep.StateBag) (string, error) {
 	return func(state multistep.StateBag) (string, error) {
 		for j := 0; j < 2; j++ {
 			var host string
@@ -28,7 +28,7 @@ func SSHAddress(e *ec2.EC2, port int, private bool) func(multistep.StateBag) (st
 			}
 
 			if host != "" {
-				return fmt.Sprintf("%s:%d", host, port), nil
+				return host, nil
 			}
 
 			r, err := e.DescribeInstances(&ec2.DescribeInstancesInput{
diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go
index a9adcf208..cd3cd8f05 100644
--- a/builder/amazon/ebs/builder.go
+++ b/builder/amazon/ebs/builder.go
@@ -115,9 +115,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		&communicator.StepConnect{
 			Config: &b.config.RunConfig.Comm,
-			SSHAddress: awscommon.SSHAddress(
+			Host: awscommon.SSHHost(
 				ec2conn,
-				b.config.RunConfig.Comm.SSHPort,
 				b.config.SSHPrivateIp),
 			SSHConfig: awscommon.SSHConfig(
 				b.config.RunConfig.Comm.SSHUsername),
diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go
index 09bda686a..d26cc63e3 100644
--- a/builder/amazon/instance/builder.go
+++ b/builder/amazon/instance/builder.go
@@ -200,9 +200,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		&communicator.StepConnect{
 			Config: &b.config.RunConfig.Comm,
-			SSHAddress: awscommon.SSHAddress(
+			Host: awscommon.SSHHost(
 				ec2conn,
-				b.config.RunConfig.Comm.SSHPort,
 				b.config.SSHPrivateIp),
 			SSHConfig: awscommon.SSHConfig(
 				b.config.RunConfig.Comm.SSHUsername),
diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go
index d5f1b7a83..ec57ed2b2 100644
--- a/builder/digitalocean/builder.go
+++ b/builder/digitalocean/builder.go
@@ -54,9 +54,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		new(stepCreateDroplet),
 		new(stepDropletInfo),
 		&communicator.StepConnect{
-			Config:     &b.config.Comm,
-			SSHAddress: sshAddress,
-			SSHConfig:  sshConfig,
+			Config:    &b.config.Comm,
+			Host:      commHost,
+			SSHConfig: sshConfig,
 		},
 		new(common.StepProvision),
 		new(stepShutdown),
diff --git a/builder/digitalocean/ssh.go b/builder/digitalocean/ssh.go
index 4d8d1f08c..5367dde1f 100644
--- a/builder/digitalocean/ssh.go
+++ b/builder/digitalocean/ssh.go
@@ -2,14 +2,14 @@ package digitalocean
 
 import (
 	"fmt"
-	"github.com/mitchellh/multistep"
 	"golang.org/x/crypto/ssh"
+
+	"github.com/mitchellh/multistep"
 )
 
-func sshAddress(state multistep.StateBag) (string, error) {
-	config := state.Get("config").(Config)
+func commHost(state multistep.StateBag) (string, error) {
 	ipAddress := state.Get("droplet_ip").(string)
-	return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil
+	return ipAddress, nil
 }
 
 func sshConfig(state multistep.StateBag) (*ssh.ClientConfig, error) {
diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go
index 3ed576b50..0f6e4bd2f 100644
--- a/builder/googlecompute/builder.go
+++ b/builder/googlecompute/builder.go
@@ -62,9 +62,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Debug: b.config.PackerDebug,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.Comm,
-			SSHAddress: sshAddress,
-			SSHConfig:  sshConfig,
+			Config:    &b.config.Comm,
+			Host:      commHost,
+			SSHConfig: sshConfig,
 		},
 		new(common.StepProvision),
 		new(StepTeardownInstance),
diff --git a/builder/googlecompute/ssh.go b/builder/googlecompute/ssh.go
index 446648884..5b7940591 100644
--- a/builder/googlecompute/ssh.go
+++ b/builder/googlecompute/ssh.go
@@ -2,15 +2,14 @@ package googlecompute
 
 import (
 	"fmt"
+
 	"github.com/mitchellh/multistep"
 	"golang.org/x/crypto/ssh"
 )
 
-// sshAddress returns the ssh address.
-func sshAddress(state multistep.StateBag) (string, error) {
-	config := state.Get("config").(*Config)
+func commHost(state multistep.StateBag) (string, error) {
 	ipAddress := state.Get("instance_ip").(string)
-	return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil
+	return ipAddress, nil
 }
 
 // sshConfig returns the ssh configuration.
diff --git a/builder/null/builder.go b/builder/null/builder.go
index 925075ee0..fed303540 100644
--- a/builder/null/builder.go
+++ b/builder/null/builder.go
@@ -30,8 +30,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 	steps := []multistep.Step{
 		&communicator.StepConnect{
 			Config: &b.config.CommConfig,
-			SSHAddress: SSHAddress(
-				b.config.CommConfig.SSHHost, b.config.CommConfig.SSHPort),
+			Host:   CommHost(b.config.CommConfig.SSHHost),
 			SSHConfig: SSHConfig(
 				b.config.CommConfig.SSHUsername,
 				b.config.CommConfig.SSHPassword,
diff --git a/builder/null/ssh.go b/builder/null/ssh.go
index e6ac9ab16..483390e86 100644
--- a/builder/null/ssh.go
+++ b/builder/null/ssh.go
@@ -8,11 +8,9 @@ import (
 	"io/ioutil"
 )
 
-// SSHAddress returns a function that can be given to the SSH communicator
-// for determining the SSH address
-func SSHAddress(host string, port int) func(multistep.StateBag) (string, error) {
+func CommHost(host string) func(multistep.StateBag) (string, error) {
 	return func(state multistep.StateBag) (string, error) {
-		return fmt.Sprintf("%s:%d", host, port), nil
+		return host, nil
 	}
 }
 
diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go
index 6d178c6ef..69ab6a016 100644
--- a/builder/openstack/builder.go
+++ b/builder/openstack/builder.go
@@ -92,10 +92,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		&communicator.StepConnect{
 			Config: &b.config.RunConfig.Comm,
-			SSHAddress: SSHAddress(
+			Host: CommHost(
 				computeClient,
-				b.config.SSHInterface,
-				b.config.RunConfig.Comm.SSHPort),
+				b.config.SSHInterface),
 			SSHConfig: SSHConfig(b.config.RunConfig.Comm.SSHUsername),
 		},
 		&common.StepProvision{},
diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go
index 76c2686b1..dc4c91771 100644
--- a/builder/openstack/ssh.go
+++ b/builder/openstack/ssh.go
@@ -13,22 +13,21 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-// SSHAddress returns a function that can be given to the SSH communicator
-// for determining the SSH address based on the server AccessIPv4 setting..
-func SSHAddress(
+// CommHost looks up the host for the communicator.
+func CommHost(
 	client *gophercloud.ServiceClient,
-	sshinterface string, port int) func(multistep.StateBag) (string, error) {
+	sshinterface string) func(multistep.StateBag) (string, error) {
 	return func(state multistep.StateBag) (string, error) {
 		s := state.Get("server").(*servers.Server)
 
 		// If we have a floating IP, use that
 		ip := state.Get("access_ip").(*floatingip.FloatingIP)
 		if ip != nil && ip.IP != "" {
-			return fmt.Sprintf("%s:%d", ip.IP, port), nil
+			return ip.IP, nil
 		}
 
 		if s.AccessIPv4 != "" {
-			return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil
+			return s.AccessIPv4, nil
 		}
 
 		// Get all the addresses associated with this server. This
@@ -53,7 +52,7 @@ func SSHAddress(
 					}
 				}
 				if addr != "" {
-					return fmt.Sprintf("%s:%d", addr, port), nil
+					return addr, nil
 				}
 			}
 		}
diff --git a/builder/parallels/common/ssh.go b/builder/parallels/common/ssh.go
index 827124677..9e0b2b907 100644
--- a/builder/parallels/common/ssh.go
+++ b/builder/parallels/common/ssh.go
@@ -1,15 +1,13 @@
 package common
 
 import (
-	"fmt"
-
 	"github.com/mitchellh/multistep"
 	commonssh "github.com/mitchellh/packer/common/ssh"
 	packerssh "github.com/mitchellh/packer/communicator/ssh"
 	"golang.org/x/crypto/ssh"
 )
 
-func SSHAddress(state multistep.StateBag) (string, error) {
+func CommHost(state multistep.StateBag) (string, error) {
 	vmName := state.Get("vmName").(string)
 	driver := state.Get("driver").(Driver)
 
@@ -23,7 +21,7 @@ func SSHAddress(state multistep.StateBag) (string, error) {
 		return "", err
 	}
 
-	return fmt.Sprintf("%s:22", ip), nil
+	return ip, nil
 }
 
 func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*ssh.ClientConfig, error) {
diff --git a/builder/parallels/iso/builder.go b/builder/parallels/iso/builder.go
index 9758e6bec..46fa73687 100644
--- a/builder/parallels/iso/builder.go
+++ b/builder/parallels/iso/builder.go
@@ -247,9 +247,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:            b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: parallelscommon.SSHAddress,
-			SSHConfig:  parallelscommon.SSHConfigFunc(b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      parallelscommon.CommHost,
+			SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig),
 		},
 		&parallelscommon.StepUploadVersion{
 			Path: b.config.PrlctlVersionFile,
diff --git a/builder/parallels/pvm/builder.go b/builder/parallels/pvm/builder.go
index 1ddd64202..0e71ea0f2 100644
--- a/builder/parallels/pvm/builder.go
+++ b/builder/parallels/pvm/builder.go
@@ -83,9 +83,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:            b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: parallelscommon.SSHAddress,
-			SSHConfig:  parallelscommon.SSHConfigFunc(b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      parallelscommon.CommHost,
+			SSHConfig: parallelscommon.SSHConfigFunc(b.config.SSHConfig),
 		},
 		&parallelscommon.StepUploadVersion{
 			Path: b.config.PrlctlVersionFile,
diff --git a/builder/qemu/builder.go b/builder/qemu/builder.go
index 2a1a2fe1c..9df908989 100644
--- a/builder/qemu/builder.go
+++ b/builder/qemu/builder.go
@@ -393,9 +393,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		&stepBootWait{},
 		&stepTypeBootCommand{},
 		&communicator.StepConnect{
-			Config:     &b.config.Comm,
-			SSHAddress: sshAddress,
-			SSHConfig:  sshConfig,
+			Config:    &b.config.Comm,
+			Host:      commHost,
+			SSHConfig: sshConfig,
+			SSHPort:   commPort,
 		},
 		new(common.StepProvision),
 		new(stepShutdown),
diff --git a/builder/qemu/ssh.go b/builder/qemu/ssh.go
index 17dc36de9..498d3fbe9 100644
--- a/builder/qemu/ssh.go
+++ b/builder/qemu/ssh.go
@@ -1,17 +1,19 @@
 package qemu
 
 import (
-	"fmt"
-
 	"github.com/mitchellh/multistep"
 	commonssh "github.com/mitchellh/packer/common/ssh"
 	"github.com/mitchellh/packer/communicator/ssh"
 	gossh "golang.org/x/crypto/ssh"
 )
 
-func sshAddress(state multistep.StateBag) (string, error) {
+func commHost(state multistep.StateBag) (string, error) {
+	return "127.0.0.1", nil
+}
+
+func commPort(state multistep.StateBag) (int, error) {
 	sshHostPort := state.Get("sshHostPort").(uint)
-	return fmt.Sprintf("127.0.0.1:%d", sshHostPort), nil
+	return int(sshHostPort), nil
 }
 
 func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) {
diff --git a/builder/virtualbox/common/ssh.go b/builder/virtualbox/common/ssh.go
index c20ac2836..2584528dd 100644
--- a/builder/virtualbox/common/ssh.go
+++ b/builder/virtualbox/common/ssh.go
@@ -1,17 +1,19 @@
 package common
 
 import (
-	"fmt"
-
 	"github.com/mitchellh/multistep"
 	commonssh "github.com/mitchellh/packer/common/ssh"
 	"github.com/mitchellh/packer/communicator/ssh"
 	gossh "golang.org/x/crypto/ssh"
 )
 
-func SSHAddress(state multistep.StateBag) (string, error) {
+func CommHost(state multistep.StateBag) (string, error) {
+	return "127.0.0.1", nil
+}
+
+func SSHPort(state multistep.StateBag) (int, error) {
 	sshHostPort := state.Get("sshHostPort").(uint)
-	return fmt.Sprintf("127.0.0.1:%d", sshHostPort), nil
+	return int(sshHostPort), nil
 }
 
 func SSHConfigFunc(config SSHConfig) func(multistep.StateBag) (*gossh.ClientConfig, error) {
diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go
index 226de0527..f37c65c5a 100644
--- a/builder/virtualbox/iso/builder.go
+++ b/builder/virtualbox/iso/builder.go
@@ -273,9 +273,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:         b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: vboxcommon.SSHAddress,
-			SSHConfig:  vboxcommon.SSHConfigFunc(b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      vboxcommon.CommHost,
+			SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig),
+			SSHPort:   vboxcommon.SSHPort,
 		},
 		&vboxcommon.StepUploadVersion{
 			Path: b.config.VBoxVersionFile,
diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go
index 05ec6159f..c35f2a50f 100644
--- a/builder/virtualbox/ovf/builder.go
+++ b/builder/virtualbox/ovf/builder.go
@@ -102,9 +102,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:         b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: vboxcommon.SSHAddress,
-			SSHConfig:  vboxcommon.SSHConfigFunc(b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      vboxcommon.CommHost,
+			SSHConfig: vboxcommon.SSHConfigFunc(b.config.SSHConfig),
+			SSHPort:   vboxcommon.SSHPort,
 		},
 		&vboxcommon.StepUploadVersion{
 			Path: b.config.VBoxVersionFile,
diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go
index ee8dbc30e..c06c8fd92 100644
--- a/builder/vmware/common/driver.go
+++ b/builder/vmware/common/driver.go
@@ -29,9 +29,9 @@ type Driver interface {
 	// Checks if the VMX file at the given path is running.
 	IsRunning(string) (bool, error)
 
-	// SSHAddress returns the SSH address for the VM that is being
+	// CommHost returns the host address for the VM that is being
 	// managed by this driver.
-	SSHAddress(multistep.StateBag) (string, error)
+	CommHost(multistep.StateBag) (string, error)
 
 	// Start starts a VM specified by the path to the VMX given.
 	Start(string, bool) error
diff --git a/builder/vmware/common/driver_fusion5.go b/builder/vmware/common/driver_fusion5.go
index 3a295e731..a10f11902 100644
--- a/builder/vmware/common/driver_fusion5.go
+++ b/builder/vmware/common/driver_fusion5.go
@@ -69,8 +69,8 @@ func (d *Fusion5Driver) IsRunning(vmxPath string) (bool, error) {
 	return false, nil
 }
 
-func (d *Fusion5Driver) SSHAddress(state multistep.StateBag) (string, error) {
-	return SSHAddressFunc(d.SSHConfig)(state)
+func (d *Fusion5Driver) CommHost(state multistep.StateBag) (string, error) {
+	return CommHost(d.SSHConfig)(state)
 }
 
 func (d *Fusion5Driver) Start(vmxPath string, headless bool) error {
diff --git a/builder/vmware/common/driver_mock.go b/builder/vmware/common/driver_mock.go
index 6aa0d02c0..fcd80a51b 100644
--- a/builder/vmware/common/driver_mock.go
+++ b/builder/vmware/common/driver_mock.go
@@ -29,10 +29,10 @@ type DriverMock struct {
 	IsRunningResult bool
 	IsRunningErr    error
 
-	SSHAddressCalled bool
-	SSHAddressState  multistep.StateBag
-	SSHAddressResult string
-	SSHAddressErr    error
+	CommHostCalled bool
+	CommHostState  multistep.StateBag
+	CommHostResult string
+	CommHostErr    error
 
 	StartCalled   bool
 	StartPath     string
@@ -92,10 +92,10 @@ func (d *DriverMock) IsRunning(path string) (bool, error) {
 	return d.IsRunningResult, d.IsRunningErr
 }
 
-func (d *DriverMock) SSHAddress(state multistep.StateBag) (string, error) {
-	d.SSHAddressCalled = true
-	d.SSHAddressState = state
-	return d.SSHAddressResult, d.SSHAddressErr
+func (d *DriverMock) CommHost(state multistep.StateBag) (string, error) {
+	d.CommHostCalled = true
+	d.CommHostState = state
+	return d.CommHostResult, d.CommHostErr
 }
 
 func (d *DriverMock) Start(path string, headless bool) error {
diff --git a/builder/vmware/common/driver_player5.go b/builder/vmware/common/driver_player5.go
index 5bb80a0f2..1552e92ea 100644
--- a/builder/vmware/common/driver_player5.go
+++ b/builder/vmware/common/driver_player5.go
@@ -97,8 +97,8 @@ func (d *Player5Driver) IsRunning(vmxPath string) (bool, error) {
 	return false, nil
 }
 
-func (d *Player5Driver) SSHAddress(state multistep.StateBag) (string, error) {
-	return SSHAddressFunc(d.SSHConfig)(state)
+func (d *Player5Driver) CommHost(state multistep.StateBag) (string, error) {
+	return CommHost(d.SSHConfig)(state)
 }
 
 func (d *Player5Driver) Start(vmxPath string, headless bool) error {
diff --git a/builder/vmware/common/driver_workstation9.go b/builder/vmware/common/driver_workstation9.go
index 4c72c72b3..debcefcbc 100644
--- a/builder/vmware/common/driver_workstation9.go
+++ b/builder/vmware/common/driver_workstation9.go
@@ -70,8 +70,8 @@ func (d *Workstation9Driver) IsRunning(vmxPath string) (bool, error) {
 	return false, nil
 }
 
-func (d *Workstation9Driver) SSHAddress(state multistep.StateBag) (string, error) {
-	return SSHAddressFunc(d.SSHConfig)(state)
+func (d *Workstation9Driver) CommHost(state multistep.StateBag) (string, error) {
+	return CommHost(d.SSHConfig)(state)
 }
 
 func (d *Workstation9Driver) Start(vmxPath string, headless bool) error {
diff --git a/builder/vmware/common/ssh.go b/builder/vmware/common/ssh.go
index 7181e90bf..86e184bb5 100644
--- a/builder/vmware/common/ssh.go
+++ b/builder/vmware/common/ssh.go
@@ -13,13 +13,13 @@ import (
 	gossh "golang.org/x/crypto/ssh"
 )
 
-func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error) {
+func CommHost(config *SSHConfig) func(multistep.StateBag) (string, error) {
 	return func(state multistep.StateBag) (string, error) {
 		driver := state.Get("driver").(Driver)
 		vmxPath := state.Get("vmx_path").(string)
 
 		if config.Comm.SSHHost != "" {
-			return fmt.Sprintf("%s:%d", config.Comm.SSHHost, config.Comm.SSHPort), nil
+			return config.Comm.SSHHost, nil
 		}
 
 		log.Println("Lookup up IP information...")
@@ -62,7 +62,7 @@ func SSHAddressFunc(config *SSHConfig) func(multistep.StateBag) (string, error)
 		}
 
 		log.Printf("Detected IP: %s", ipAddress)
-		return fmt.Sprintf("%s:%d", ipAddress, config.Comm.SSHPort), nil
+		return ipAddress, nil
 	}
 }
 
diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go
index 9599df675..38ba3a4a1 100644
--- a/builder/vmware/iso/builder.go
+++ b/builder/vmware/iso/builder.go
@@ -300,9 +300,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:         b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: driver.SSHAddress,
-			SSHConfig:  vmwcommon.SSHConfigFunc(&b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      driver.CommHost,
+			SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig),
 		},
 		&vmwcommon.StepUploadTools{
 			RemoteType:        b.config.RemoteType,
diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go
index 26642f3d7..8162db468 100644
--- a/builder/vmware/iso/driver_esx5.go
+++ b/builder/vmware/iso/driver_esx5.go
@@ -218,7 +218,7 @@ func (d *ESX5Driver) VNCAddress(portMin, portMax uint) (string, uint, error) {
 	return d.Host, vncPort, nil
 }
 
-func (d *ESX5Driver) SSHAddress(state multistep.StateBag) (string, error) {
+func (d *ESX5Driver) CommHost(state multistep.StateBag) (string, error) {
 	config := state.Get("config").(*Config)
 
 	if address, ok := state.GetOk("vm_address"); ok {
@@ -253,7 +253,7 @@ func (d *ESX5Driver) SSHAddress(state multistep.StateBag) (string, error) {
 		return "", errors.New("VM network port found, but no IP address")
 	}
 
-	address := fmt.Sprintf("%s:%d", record["IPAddress"], config.Comm.SSHPort)
+	address := record["IPAddress"]
 	state.Put("vm_address", address)
 	return address, nil
 }
diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go
index 1390da547..aa86d3669 100644
--- a/builder/vmware/vmx/builder.go
+++ b/builder/vmware/vmx/builder.go
@@ -92,9 +92,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			Ctx:         b.config.ctx,
 		},
 		&communicator.StepConnect{
-			Config:     &b.config.SSHConfig.Comm,
-			SSHAddress: driver.SSHAddress,
-			SSHConfig:  vmwcommon.SSHConfigFunc(&b.config.SSHConfig),
+			Config:    &b.config.SSHConfig.Comm,
+			Host:      driver.CommHost,
+			SSHConfig: vmwcommon.SSHConfigFunc(&b.config.SSHConfig),
 		},
 		&vmwcommon.StepUploadTools{
 			RemoteType:        b.config.RemoteType,
diff --git a/helper/communicator/step_connect.go b/helper/communicator/step_connect.go
index e6338027e..a31dd4eb8 100644
--- a/helper/communicator/step_connect.go
+++ b/helper/communicator/step_connect.go
@@ -15,15 +15,16 @@ type StepConnect struct {
 	// Config is the communicator config struct
 	Config *Config
 
+	// Host should return a host that can be connected to for communicator
+	// connections.
+	Host func(multistep.StateBag) (string, error)
+
 	// The fields below are callbacks to assist with connecting to SSH.
 	//
-	// SSHAddress should return the default host to connect to for SSH.
-	// This is only called if ssh_host isn't specified in the config.
-	//
 	// SSHConfig should return the default configuration for
 	// connecting via SSH.
-	SSHAddress func(multistep.StateBag) (string, error)
-	SSHConfig  func(multistep.StateBag) (*gossh.ClientConfig, error)
+	SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error)
+	SSHPort   func(multistep.StateBag) (int, error)
 
 	substep multistep.Step
 }
@@ -32,9 +33,10 @@ func (s *StepConnect) Run(state multistep.StateBag) multistep.StepAction {
 	typeMap := map[string]multistep.Step{
 		"none": nil,
 		"ssh": &StepConnectSSH{
-			Config:     s.Config,
-			SSHAddress: s.SSHAddress,
-			SSHConfig:  s.SSHConfig,
+			Config:    s.Config,
+			Host:      s.Host,
+			SSHConfig: s.SSHConfig,
+			SSHPort:   s.SSHPort,
 		},
 	}
 
diff --git a/helper/communicator/step_connect_ssh.go b/helper/communicator/step_connect_ssh.go
index 9be653c01..0b54bae9d 100644
--- a/helper/communicator/step_connect_ssh.go
+++ b/helper/communicator/step_connect_ssh.go
@@ -18,9 +18,10 @@ import (
 // In general, you should use StepConnect.
 type StepConnectSSH struct {
 	// All the fields below are documented on StepConnect
-	Config     *Config
-	SSHAddress func(multistep.StateBag) (string, error)
-	SSHConfig  func(multistep.StateBag) (*gossh.ClientConfig, error)
+	Config    *Config
+	Host      func(multistep.StateBag) (string, error)
+	SSHConfig func(multistep.StateBag) (*gossh.ClientConfig, error)
+	SSHPort   func(multistep.StateBag) (int, error)
 }
 
 func (s *StepConnectSSH) Run(state multistep.StateBag) multistep.StepAction {
@@ -95,11 +96,19 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru
 		first = false
 
 		// First we request the TCP connection information
-		address, err := s.SSHAddress(state)
+		host, err := s.Host(state)
 		if err != nil {
 			log.Printf("[DEBUG] Error getting SSH address: %s", err)
 			continue
 		}
+		port := s.Config.SSHPort
+		if s.SSHPort != nil {
+			port, err = s.SSHPort(state)
+			if err != nil {
+				log.Printf("[DEBUG] Error getting SSH port: %s", err)
+				continue
+			}
+		}
 
 		// Retrieve the SSH configuration
 		sshConfig, err := s.SSHConfig(state)
@@ -108,6 +117,8 @@ func (s *StepConnectSSH) waitForSSH(state multistep.StateBag, cancel <-chan stru
 			continue
 		}
 
+		address := fmt.Sprintf("%s:%d", host, port)
+
 		// Attempt to connect to SSH port
 		connFunc := ssh.ConnectFunc("tcp", address)
 		nc, err := connFunc()