diff --git a/builder/azure/arm/config.go b/builder/azure/arm/config.go
index c7348d2ee..1a2485c9e 100644
--- a/builder/azure/arm/config.go
+++ b/builder/azure/arm/config.go
@@ -248,11 +248,11 @@ func setSshValues(c *Config) error {
 	if c.Comm.SSHPrivateKey != "" {
 		privateKeyBytes, err := ioutil.ReadFile(c.Comm.SSHPrivateKey)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		signer, err := ssh.ParsePrivateKey(privateKeyBytes)
 		if err != nil {
-			panic(err)
+			return err
 		}
 
 		publicKey := signer.PublicKey()