Merge pull request #9726 from acornies/feature/salt-masterless-formulas

Feature: salt-masterless formulas

provisioner/salt-masterless/provisioner.go

@@ -11,8 +11,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	"github.com/hashicorp/go-getter/v2"
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
@@ -72,6 +74,9 @@ type Config struct {
 	// The Guest OS Type (unix or windows)
 	GuestOSType string `mapstructure:"guest_os_type"`
 
+	// An array of private or community git source formulas
+	Formulas []string `mapstructure:"formulas"`
+
 	ctx interpolate.Context
 }
 
@@ -152,6 +157,14 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 		errs = packer.MultiErrorAppend(errs, err)
 	}
 
+	if p.config.Formulas != nil && len(p.config.Formulas) > 0 {
+
+		validURLs := hasValidFormulaURLs(p.config.Formulas)
+		if !validURLs {
+			errs = packer.MultiErrorAppend(errs, fmt.Errorf("Invalid formula URL. Please verify the git URLs also contain a '//' subdir"))
+		}
+	}
+
 	err = validateDirConfig(p.config.LocalPillarRoots, "local_pillar_roots", false)
 	if err != nil {
 		errs = packer.MultiErrorAppend(errs, err)
@@ -228,6 +241,35 @@ func (p *Provisioner) Prepare(raws ...interface{}) error {
 func (p *Provisioner) Provision(ctx context.Context, ui packer.Ui, comm packer.Communicator, _ map[string]interface{}) error {
 	var err error
 	var src, dst string
+	var formulas []string
+
+	if p.config.Formulas != nil && len(p.config.Formulas) > 0 {
+		ui.Say("Downloading Salt formulas...")
+		client := new(getter.Client)
+		for _, i := range p.config.Formulas {
+			req := getter.Request{
+				Src: i,
+			}
+			// Use //subdirectory name when creating in local_state_tree directory
+			state := strings.Split(i, "//")
+			last := state[len(state)-1]
+			path := filepath.Join(p.config.LocalStateTree, last)
+			formulas = append(formulas, path)
+			if _, err := os.Stat(path); os.IsNotExist(err) {
+				ui.Message(fmt.Sprintf("%s => %s", i, path))
+				if err = os.Mkdir(path, 0755); err != nil {
+					return fmt.Errorf("Unable to create Salt state directory: %s", err)
+				}
+				req.Dst = path
+				req.Mode = getter.ModeAny
+				if _, err := client.Get(ctx, &req); err != nil {
+					return fmt.Errorf("Unable to download Salt formula from %s: %s", i, err)
+				}
+			} else {
+				ui.Message(fmt.Sprintf("Found existing formula at: %s", path))
+			}
+		}
+	}
 
 	ui.Say("Provisioning with Salt...")
 	if !p.config.SkipBootstrap {
@@ -318,6 +360,16 @@ func (p *Provisioner) Provision(ctx context.Context, ui packer.Ui, comm packer.C
 		return fmt.Errorf("Unable to move %s/states to %s: %s", p.config.TempConfigDir, dst, err)
 	}
 
+	// Remove the local Salt formulas if present
+	if p.config.Formulas != nil {
+		for _, f := range formulas {
+			if _, err := os.Stat(f); !os.IsNotExist(err) && f != p.config.LocalStateTree {
+				ui.Message(fmt.Sprintf("Removing Salt formula: %s", f))
+				defer os.RemoveAll(f)
+			}
+		}
+	}
+
 	if p.config.LocalPillarRoots != "" {
 		ui.Message(fmt.Sprintf("Uploading local pillar roots: %s", p.config.LocalPillarRoots))
 		src = p.config.LocalPillarRoots
@@ -397,6 +449,18 @@ func validateFileConfig(path string, name string, required bool) error {
 	return nil
 }
 
+func hasValidFormulaURLs(s []string) bool {
+	re := regexp.MustCompile(`^(.*).git\/\/[a-zA-Z0-9-_]+(\?.*)?$`)
+
+	for _, u := range s {
+		if !re.MatchString(u) {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (p *Provisioner) uploadFile(ui packer.Ui, comm packer.Communicator, dst, src string) error {
 	f, err := os.Open(src)
 	if err != nil {
@@ -410,7 +474,10 @@ func (p *Provisioner) uploadFile(ui packer.Ui, comm packer.Communicator, dst, sr
 		return fmt.Errorf("Error uploading %s: %s", src, err)
 	}
 
-	p.moveFile(ui, comm, dst, temp_dst)
+	err = p.moveFile(ui, comm, dst, temp_dst)
+	if err != nil {
+		return fmt.Errorf("Error moving file to destination: %s", err)
+	}
 
 	return nil
 }

provisioner/salt-masterless/provisioner.hcl2spec.go

@@ -32,6 +32,7 @@ type FlatConfig struct {
 	SaltCallArgs        *string           `mapstructure:"salt_call_args" cty:"salt_call_args" hcl:"salt_call_args"`
 	SaltBinDir          *string           `mapstructure:"salt_bin_dir" cty:"salt_bin_dir" hcl:"salt_bin_dir"`
 	GuestOSType         *string           `mapstructure:"guest_os_type" cty:"guest_os_type" hcl:"guest_os_type"`
+	Formulas            []string          `mapstructure:"formulas" cty:"formulas" hcl:"formulas"`
 }
 
 // FlatMapstructure returns a new FlatConfig.
@@ -69,6 +70,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"salt_call_args":             &hcldec.AttrSpec{Name: "salt_call_args", Type: cty.String, Required: false},
 		"salt_bin_dir":               &hcldec.AttrSpec{Name: "salt_bin_dir", Type: cty.String, Required: false},
 		"guest_os_type":              &hcldec.AttrSpec{Name: "guest_os_type", Type: cty.String, Required: false},
+		"formulas":                   &hcldec.AttrSpec{Name: "formulas", Type: cty.List(cty.String), Required: false},
 	}
 	return s
 }

provisioner/salt-masterless/provisioner_test.go

@@ -325,3 +325,34 @@ func TestProvisionerPrepare_GuestOSType(t *testing.T) {
 		t.Fatalf("GuestOSType should be 'windows'")
 	}
 }
+
+func TestProvisionerPrepare_BadFormulaURL(t *testing.T) {
+	var p Provisioner
+	config := testConfig()
+
+	config["formulas"] = []string{
+		"git::https://github.com/org/some-formula.git//",
+	}
+
+	err := p.Prepare(config)
+	if err == nil {
+		t.Fatalf("Expected invalid formula URL: %s", err)
+	}
+}
+
+func TestProvisionerPrepare_ValidFormulaURLs(t *testing.T) {
+
+	var p Provisioner
+	config := testConfig()
+
+	config["formulas"] = []string{
+		"git::https://github.com/org/some-formula.git//example",
+		"git@github.com:org/some-formula.git//example",
+		"git::https://github.com/org/some-formula.git//example?ref=example",
+	}
+
+	err := p.Prepare(config)
+	if err != nil {
+		t.Fatalf("Unexpected error in formula URLs: %s", err)
+	}
+}

website/pages/docs/provisioners/salt-masterless.mdx

@@ -100,4 +100,9 @@ Optional:
 - `guest_os_type` (string) - The target guest OS type, either "unix" or
   "windows".
 
+- `formulas` (array of strings) - An array of git source formulas to be downloaded to the local
+  state tree prior to moving to the remote state tree. Note: `//directory` must be included in 
+  the URL to download the appropriate formula directory. Example:
+  `git::https://github.com/saltstack-formulas/vault-formula.git//vault?ref=v1.2.3`
+
 @include 'provisioners/common-config.mdx'