From 41d0adfbdd1599e40a862aa2e48b0bb42b9617c1 Mon Sep 17 00:00:00 2001 From: Zbigniew Kostrzewa Date: Wed, 28 Jun 2017 12:33:53 +0200 Subject: [PATCH 01/11] Add playbook_files to execute multiple ansible playbooks. --- provisioner/ansible-local/provisioner.go | 120 ++++++++++++++++-- provisioner/ansible-local/provisioner_test.go | 44 +++++++ 2 files changed, 150 insertions(+), 14 deletions(-) diff --git a/provisioner/ansible-local/provisioner.go b/provisioner/ansible-local/provisioner.go index 2f6a873f9..a25aca36c 100644 --- a/provisioner/ansible-local/provisioner.go +++ b/provisioner/ansible-local/provisioner.go @@ -38,6 +38,9 @@ type Config struct { // The main playbook file to execute. PlaybookFile string `mapstructure:"playbook_file"` + // The playbook files to execute. + PlaybookFiles []string `mapstructure:"playbook_files"` + // An array of local paths of playbook files to upload. PlaybookPaths []string `mapstructure:"playbook_paths"` @@ -63,6 +66,8 @@ type Config struct { type Provisioner struct { config Config + + playbookFiles []string } func (p *Provisioner) Prepare(raws ...interface{}) error { @@ -77,6 +82,9 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { return err } + // Reset the state. + p.playbookFiles = make([]string, 0, len(p.config.PlaybookFiles)) + // Defaults if p.config.Command == "" { p.config.Command = "ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook" @@ -91,9 +99,32 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { // Validation var errs *packer.MultiError - err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true) - if err != nil { - errs = packer.MultiErrorAppend(errs, err) + + // Check that either playbook_file or playbook_files is specified + if len(p.config.PlaybookFiles) != 0 && p.config.PlaybookFile != "" { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("Either playbook_file or playbook_files can be specified, not both")) + } + if len(p.config.PlaybookFiles) == 0 && p.config.PlaybookFile == "" { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("Either playbook_file or playbook_files must be specified")) + } + if p.config.PlaybookFile != "" { + err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true) + if err != nil { + errs = packer.MultiErrorAppend(errs, err) + } + } + + for _, playbookFile := range p.config.PlaybookFiles { + if err := validateFileConfig(playbookFile, "playbook_files", true); err != nil { + errs = packer.MultiErrorAppend(errs, err) + } else { + playbookFile, err := filepath.Abs(playbookFile) + if err != nil { + errs = packer.MultiErrorAppend(errs, err) + } else { + p.playbookFiles = append(p.playbookFiles, playbookFile) + } + } } // Check that the inventory file exists, if configured @@ -166,11 +197,15 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } } - ui.Message("Uploading main Playbook file...") - src := p.config.PlaybookFile - dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) - if err := p.uploadFile(ui, comm, dst, src); err != nil { - return fmt.Errorf("Error uploading main playbook: %s", err) + if p.config.PlaybookFile != "" { + ui.Message("Uploading main Playbook file...") + src := p.config.PlaybookFile + dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) + if err := p.uploadFile(ui, comm, dst, src); err != nil { + return fmt.Errorf("Error uploading main playbook: %s", err) + } + } else if err := p.provisionPlaybookFiles(ui, comm); err != nil { + return err } if len(p.config.InventoryFile) == 0 { @@ -201,16 +236,16 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { if len(p.config.GalaxyFile) > 0 { ui.Message("Uploading galaxy file...") - src = p.config.GalaxyFile - dst = filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) + src := p.config.GalaxyFile + dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) if err := p.uploadFile(ui, comm, dst, src); err != nil { return fmt.Errorf("Error uploading galaxy file: %s", err) } } ui.Message("Uploading inventory file...") - src = p.config.InventoryFile - dst = filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) + src := p.config.InventoryFile + dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) if err := p.uploadFile(ui, comm, dst, src); err != nil { return fmt.Errorf("Error uploading inventory file: %s", err) } @@ -269,6 +304,44 @@ func (p *Provisioner) Cancel() { os.Exit(0) } +func (p *Provisioner) provisionPlaybookFiles(ui packer.Ui, comm packer.Communicator) error { + var playbookDir string + if p.config.PlaybookDir != "" { + var err error + playbookDir, err = filepath.Abs(p.config.PlaybookDir) + if err != nil { + return err + } + } + for index, playbookFile := range p.playbookFiles { + if playbookDir != "" && strings.HasPrefix(playbookFile, playbookDir) { + p.playbookFiles[index] = strings.TrimPrefix(playbookFile, playbookDir) + continue + } + if err := p.provisionPlaybookFile(ui, comm, playbookFile); err != nil { + return err + } + } + return nil +} + +func (p *Provisioner) provisionPlaybookFile(ui packer.Ui, comm packer.Communicator, playbookFile string) error { + ui.Message(fmt.Sprintf("Uploading playbook file: %s", playbookFile)) + + remoteDir := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Dir(playbookFile))) + remotePlaybookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile)) + + if err := p.createDir(ui, comm, remoteDir); err != nil { + return fmt.Errorf("Error uploading playbook file: %s [%s]", playbookFile, err) + } + + if err := p.uploadFile(ui, comm, remotePlaybookFile, playbookFile); err != nil { + return fmt.Errorf("Error uploading playbook: %s [%s]", playbookFile, err) + } + + return nil +} + func (p *Provisioner) executeGalaxy(ui packer.Ui, comm packer.Communicator) error { rolesDir := filepath.ToSlash(filepath.Join(p.config.StagingDir, "roles")) galaxyFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.GalaxyFile))) @@ -291,7 +364,6 @@ func (p *Provisioner) executeGalaxy(ui packer.Ui, comm packer.Communicator) erro } func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) error { - playbook := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile))) inventory := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.InventoryFile))) extraArgs := fmt.Sprintf(" --extra-vars \"packer_build_name=%s packer_builder_type=%s packer_http_addr=%s\" ", @@ -307,8 +379,28 @@ func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) err } } + if p.config.PlaybookFile != "" { + playbookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile))) + if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { + return err + } + } + + for _, playbookFile := range p.playbookFiles { + playbookFile = filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile)) + if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { + return err + } + } + return nil +} + +func (p *Provisioner) executeAnsiblePlaybook( + ui packer.Ui, comm packer.Communicator, playbookFile, extraArgs, inventory string, +) error { command := fmt.Sprintf("cd %s && %s %s%s -c local -i %s", - p.config.StagingDir, p.config.Command, playbook, extraArgs, inventory) + p.config.StagingDir, p.config.Command, playbookFile, extraArgs, inventory, + ) ui.Message(fmt.Sprintf("Executing Ansible: %s", command)) cmd := &packer.RemoteCmd{ Command: command, diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 2195b7107..a074e1331 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -73,6 +73,50 @@ func TestProvisionerPrepare_PlaybookFile(t *testing.T) { } } +func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { + var p Provisioner + config := testConfig() + + err := p.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["playbook_file"] = "" + config["playbook_files"] = []string{} + err = p.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + playbook_file, err := ioutil.TempFile("", "playbook") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(playbook_file.Name()) + + config["playbook_file"] = playbook_file.Name() + config["playbook_files"] = []string{"some_other_file"} + err = p.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["playbook_file"] = playbook_file.Name() + config["playbook_files"] = []string{} + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + config["playbook_file"] = "" + config["playbook_files"] = []string{playbook_file.Name()} + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } +} + func TestProvisionerPrepare_InventoryFile(t *testing.T) { var p Provisioner config := testConfig() From f68204534507fa7846b6b6bf477f91def3297c18 Mon Sep 17 00:00:00 2001 From: localghost Date: Mon, 3 Jul 2017 23:44:10 +0200 Subject: [PATCH 02/11] Test for ansible-local playbook_files with mocked Communicator. --- .../ansible-local/communicator_mock.go | 38 +++++++++ provisioner/ansible-local/provisioner_test.go | 84 +++++++++++++++++++ provisioner/ansible-local/ui_stub.go | 15 ++++ 3 files changed, 137 insertions(+) create mode 100644 provisioner/ansible-local/communicator_mock.go create mode 100644 provisioner/ansible-local/ui_stub.go diff --git a/provisioner/ansible-local/communicator_mock.go b/provisioner/ansible-local/communicator_mock.go new file mode 100644 index 000000000..8ed59e9bb --- /dev/null +++ b/provisioner/ansible-local/communicator_mock.go @@ -0,0 +1,38 @@ +package ansiblelocal + +import ( + "github.com/hashicorp/packer/packer" + "io" + "os" +) + +type communicatorMock struct { + startCommand []string + uploadDestination []string +} + +func (c *communicatorMock) Start(cmd *packer.RemoteCmd) error { + c.startCommand = append(c.startCommand, cmd.Command) + cmd.SetExited(0) + return nil +} + +func (c *communicatorMock) Upload(dst string, _ io.Reader, _ *os.FileInfo) error { + c.uploadDestination = append(c.uploadDestination, dst) + return nil +} + +func (c *communicatorMock) UploadDir(dst, src string, exclude []string) error { + return nil +} + +func (c *communicatorMock) Download(src string, dst io.Writer) error { + return nil +} + +func (c *communicatorMock) DownloadDir(src, dst string, exclude []string) error { + return nil +} + +func (c *communicatorMock) verify() { +} diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index a074e1331..1e81b1bd6 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "fmt" "github.com/hashicorp/packer/packer" ) @@ -15,6 +16,37 @@ func testConfig() map[string]interface{} { return m } +func createTempFile() string { + file, err := ioutil.TempFile("", "") + if err != nil { + panic(fmt.Sprintf("err: %s", err)) + } + return file.Name() +} + +func createTempFiles(numFiles int) []string { + files := make([]string, 0, numFiles) + defer func() { + // Cleanup the files if not all were created. + if len(files) < numFiles { + for _, file := range files { + os.Remove(file) + } + } + }() + + for i := 0; i < numFiles; i++ { + files = append(files, createTempFile()) + } + return files +} + +func removeFiles(files ...string) { + for _, file := range files { + os.Remove(file) + } +} + func TestProvisioner_Impl(t *testing.T) { var raw interface{} raw = &Provisioner{} @@ -117,6 +149,58 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { } } +func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { + cmdIndex := 0 + for _, playbook := range playbooks { + for ; cmdIndex < len(comm.startCommand); cmdIndex++ { + cmd := comm.startCommand[cmdIndex] + if strings.Contains(cmd, "ansible-playbook") && strings.Contains(cmd, playbook) { + break + } + } + if cmdIndex == len(comm.startCommand) { + panic(fmt.Sprintf("Playbook %s was not executed", playbook)) + } + } +} + +func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { + uploadIndex := 0 + for _, playbook := range playbooks { + for ; uploadIndex < len(comm.uploadDestination); uploadIndex++ { + dest := comm.uploadDestination[uploadIndex] + if strings.HasSuffix(dest, playbook) { + break + } + } + if uploadIndex == len(comm.uploadDestination) { + panic(fmt.Sprintf("Playbook %s was not uploaded", playbook)) + } + } +} + +func TestProvisionerProvision_PlaybookFiles(t *testing.T) { + var p Provisioner + config := testConfig() + + playbooks := createTempFiles(3) + defer removeFiles(playbooks...) + + config["playbook_files"] = playbooks + err := p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + comm := &communicatorMock{} + if err := p.Provision(&uiStub{}, comm); err != nil { + t.Fatalf("err: %s", err) + } + + assertPlaybooksUploaded(comm, playbooks) + assertPlaybooksExecuted(comm, playbooks) +} + func TestProvisionerPrepare_InventoryFile(t *testing.T) { var p Provisioner config := testConfig() diff --git a/provisioner/ansible-local/ui_stub.go b/provisioner/ansible-local/ui_stub.go new file mode 100644 index 000000000..4faa2a215 --- /dev/null +++ b/provisioner/ansible-local/ui_stub.go @@ -0,0 +1,15 @@ +package ansiblelocal + +type uiStub struct{} + +func (su *uiStub) Ask(string) (string, error) { + return "", nil +} + +func (su *uiStub) Error(string) {} + +func (su *uiStub) Machine(string, ...string) {} + +func (su *uiStub) Message(string) {} + +func (su *uiStub) Say(msg string) {} From 9ea6313b68593b0c15e351f80fdb6f5ef9e1a0ea Mon Sep 17 00:00:00 2001 From: localghost Date: Thu, 6 Jul 2017 21:02:46 +0200 Subject: [PATCH 03/11] Fix playbook_files test on Windows. --- provisioner/ansible-local/provisioner_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 1e81b1bd6..5fdcdfe8e 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -152,6 +152,7 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { cmdIndex := 0 for _, playbook := range playbooks { + playbook = filepath.ToSlash(playbook) for ; cmdIndex < len(comm.startCommand); cmdIndex++ { cmd := comm.startCommand[cmdIndex] if strings.Contains(cmd, "ansible-playbook") && strings.Contains(cmd, playbook) { @@ -167,6 +168,7 @@ func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { uploadIndex := 0 for _, playbook := range playbooks { + playbook = filepath.ToSlash(playbook) for ; uploadIndex < len(comm.uploadDestination); uploadIndex++ { dest := comm.uploadDestination[uploadIndex] if strings.HasSuffix(dest, playbook) { From 1bd32d3876df0e0a10f6d57f363e0d124d509bdc Mon Sep 17 00:00:00 2001 From: localghost Date: Sun, 9 Jul 2017 20:04:01 +0200 Subject: [PATCH 04/11] Add documentation about playbook_files option of ansible-local provisioner. --- website/source/docs/provisioners/ansible-local.html.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/website/source/docs/provisioners/ansible-local.html.md b/website/source/docs/provisioners/ansible-local.html.md index 3ca775cf4..3f9500534 100644 --- a/website/source/docs/provisioners/ansible-local.html.md +++ b/website/source/docs/provisioners/ansible-local.html.md @@ -43,7 +43,12 @@ Required: - `playbook_file` (string) - The playbook file to be executed by ansible. This file must exist on your local system and will be uploaded to the - remote machine. + remote machine. This option is exclusive with `playbook_files`. + +- `playbook_files` (array of strings) - The playbook files to be executed by ansible. + These files must exist on your local system. If the files don't exist in the `playbook_dir` + or you don't set `playbook_dir` they will be uploaded to the remote machine. This option + is exclusive with `playbook_file`. Optional: From 33ae9cb2bb9134942609d178fe5421507d817c23 Mon Sep 17 00:00:00 2001 From: Zbigniew Kostrzewa Date: Mon, 10 Jul 2017 08:19:29 +0200 Subject: [PATCH 05/11] Add test for playbook_files using docker builder. --- provisioner/ansible-local/provisioner_test.go | 175 ++++++++++++++---- .../ansible-local/test-fixtures/hello.yml | 5 + .../ansible-local/test-fixtures/world.yml | 5 + 3 files changed, 149 insertions(+), 36 deletions(-) create mode 100644 provisioner/ansible-local/test-fixtures/hello.yml create mode 100644 provisioner/ansible-local/test-fixtures/world.yml diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 5fdcdfe8e..bd6568afc 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -8,45 +8,13 @@ import ( "testing" "fmt" + "github.com/hashicorp/packer/builder/docker" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/provisioner/file" + "github.com/hashicorp/packer/template" + "os/exec" ) -func testConfig() map[string]interface{} { - m := make(map[string]interface{}) - return m -} - -func createTempFile() string { - file, err := ioutil.TempFile("", "") - if err != nil { - panic(fmt.Sprintf("err: %s", err)) - } - return file.Name() -} - -func createTempFiles(numFiles int) []string { - files := make([]string, 0, numFiles) - defer func() { - // Cleanup the files if not all were created. - if len(files) < numFiles { - for _, file := range files { - os.Remove(file) - } - } - }() - - for i := 0; i < numFiles; i++ { - files = append(files, createTempFile()) - } - return files -} - -func removeFiles(files ...string) { - for _, file := range files { - os.Remove(file) - } -} - func TestProvisioner_Impl(t *testing.T) { var raw interface{} raw = &Provisioner{} @@ -318,3 +286,138 @@ func TestProvisionerPrepare_Dirs(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestProvisionerProvisionDocker_PlaybookFiles(t *testing.T) { + if os.Getenv("PACKER_ACC") == "" { + t.Skip("This test is only run with PACKER_ACC=1") + } + + ui := packer.TestUi(t) + cache := &packer.FileCache{CacheDir: os.TempDir()} + + tpl, err := template.Parse(strings.NewReader(playbookFilesDockerConfig)) + if err != nil { + t.Fatalf("Unable to parse config: %s", err) + } + + // Check if docker executable can be found. + _, err = exec.LookPath("docker") + if err != nil { + t.Error("docker command not found; please make sure docker is installed") + } + + // Setup the builder + builder := &docker.Builder{} + warnings, err := builder.Prepare(tpl.Builders["docker"].Config) + if err != nil { + t.Fatalf("Error preparing configuration %s", err) + } + if len(warnings) > 0 { + t.Fatal("Encountered configuration warnings; aborting") + } + + ansible := &Provisioner{} + err = ansible.Prepare(tpl.Provisioners[0].Config) + if err != nil { + t.Fatalf("Error preparing ansible-local provisioner: %s", err) + } + + download := &file.Provisioner{} + err = download.Prepare(tpl.Provisioners[1].Config) + if err != nil { + t.Fatalf("Error preparing download: %s", err) + } + defer os.Remove("hello_world") + + // Add hooks so the provisioners run during the build + hooks := map[string][]packer.Hook{} + hooks[packer.HookProvision] = []packer.Hook{ + &packer.ProvisionHook{ + Provisioners: []packer.Provisioner{ + ansible, + download, + }, + ProvisionerTypes: []string{tpl.Provisioners[0].Type, tpl.Provisioners[1].Type}, + }, + } + hook := &packer.DispatchHook{Mapping: hooks} + + artifact, err := builder.Run(ui, hook, cache) + if err != nil { + t.Fatalf("Error running build %s", err) + } + defer artifact.Destroy() + + actualContent, err := ioutil.ReadFile("hello_world") + if err != nil { + t.Fatalf("Expected file not found: %s", err) + } + + expectedContent := "Hello world!" + if string(actualContent) != expectedContent { + t.Fatalf(`Unexpected file content: expected="%s", actual="%s"`, expectedContent, actualContent) + } +} + +func testConfig() map[string]interface{} { + m := make(map[string]interface{}) + return m +} + +func createTempFile() string { + file, err := ioutil.TempFile("", "") + if err != nil { + panic(fmt.Sprintf("err: %s", err)) + } + return file.Name() +} + +func createTempFiles(numFiles int) []string { + files := make([]string, 0, numFiles) + defer func() { + // Cleanup the files if not all were created. + if len(files) < numFiles { + for _, file := range files { + os.Remove(file) + } + } + }() + + for i := 0; i < numFiles; i++ { + files = append(files, createTempFile()) + } + return files +} + +func removeFiles(files ...string) { + for _, file := range files { + os.Remove(file) + } +} + +const playbookFilesDockerConfig = ` +{ + "builders": [ + { + "type": "docker", + "image": "williamyeh/ansible:centos7", + "discard": true + } + ], + "provisioners": [ + { + "type": "ansible-local", + "playbook_files": [ + "test-fixtures/hello.yml", + "test-fixtures/world.yml" + ] + }, + { + "type": "file", + "source": "/tmp/hello_world", + "destination": "hello_world", + "direction": "download" + } + ] +} +` diff --git a/provisioner/ansible-local/test-fixtures/hello.yml b/provisioner/ansible-local/test-fixtures/hello.yml new file mode 100644 index 000000000..6bb8797d8 --- /dev/null +++ b/provisioner/ansible-local/test-fixtures/hello.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + tasks: + - name: write Hello + shell: echo -n "Hello" >> /tmp/hello_world \ No newline at end of file diff --git a/provisioner/ansible-local/test-fixtures/world.yml b/provisioner/ansible-local/test-fixtures/world.yml new file mode 100644 index 000000000..98a205c7b --- /dev/null +++ b/provisioner/ansible-local/test-fixtures/world.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + tasks: + - name: write world! + shell: echo -n " world!" >> /tmp/hello_world \ No newline at end of file From 079cbc263fa631ada838d9fab11f7771e51e1b76 Mon Sep 17 00:00:00 2001 From: localghost Date: Mon, 10 Jul 2017 21:58:46 +0200 Subject: [PATCH 06/11] Add tests for playbook_files with playbook_dir. --- provisioner/ansible-local/provisioner_test.go | 161 +++++++++++++----- 1 file changed, 122 insertions(+), 39 deletions(-) diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index bd6568afc..461e53539 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/provisioner/file" "github.com/hashicorp/packer/template" + "github.com/moby/moby/pkg/ioutils" "os/exec" ) @@ -117,43 +118,11 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { } } -func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { - cmdIndex := 0 - for _, playbook := range playbooks { - playbook = filepath.ToSlash(playbook) - for ; cmdIndex < len(comm.startCommand); cmdIndex++ { - cmd := comm.startCommand[cmdIndex] - if strings.Contains(cmd, "ansible-playbook") && strings.Contains(cmd, playbook) { - break - } - } - if cmdIndex == len(comm.startCommand) { - panic(fmt.Sprintf("Playbook %s was not executed", playbook)) - } - } -} - -func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { - uploadIndex := 0 - for _, playbook := range playbooks { - playbook = filepath.ToSlash(playbook) - for ; uploadIndex < len(comm.uploadDestination); uploadIndex++ { - dest := comm.uploadDestination[uploadIndex] - if strings.HasSuffix(dest, playbook) { - break - } - } - if uploadIndex == len(comm.uploadDestination) { - panic(fmt.Sprintf("Playbook %s was not uploaded", playbook)) - } - } -} - func TestProvisionerProvision_PlaybookFiles(t *testing.T) { var p Provisioner config := testConfig() - playbooks := createTempFiles(3) + playbooks := createTempFiles("", 3) defer removeFiles(playbooks...) config["playbook_files"] = playbooks @@ -171,6 +140,40 @@ func TestProvisionerProvision_PlaybookFiles(t *testing.T) { assertPlaybooksExecuted(comm, playbooks) } +func TestProvisionerProvision_PlaybookFilesWithPlaybookDir(t *testing.T) { + var p Provisioner + config := testConfig() + + playbook_dir, err := ioutils.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create playbook_dir: %s", err) + } + defer os.RemoveAll(playbook_dir) + playbooks := createTempFiles(playbook_dir, 3) + + playbookNames := make([]string, 0, len(playbooks)) + playbooksInPlaybookDir := make([]string, 0, len(playbooks)) + for _, playbook := range playbooks { + playbooksInPlaybookDir = append(playbooksInPlaybookDir, strings.TrimPrefix(playbook, playbook_dir)) + playbookNames = append(playbookNames, filepath.Base(playbook)) + } + + config["playbook_files"] = playbooks + config["playbook_dir"] = playbook_dir + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + comm := &communicatorMock{} + if err := p.Provision(&uiStub{}, comm); err != nil { + t.Fatalf("err: %s", err) + } + + assertPlaybooksNotUploaded(comm, playbookNames) + assertPlaybooksExecuted(comm, playbooksInPlaybookDir) +} + func TestProvisionerPrepare_InventoryFile(t *testing.T) { var p Provisioner config := testConfig() @@ -288,6 +291,14 @@ func TestProvisionerPrepare_Dirs(t *testing.T) { } func TestProvisionerProvisionDocker_PlaybookFiles(t *testing.T) { + testProvisionerProvisionDockerWithPlaybookFiles(t, playbookFilesDockerTemplate) +} + +func TestProvisionerProvisionDocker_PlaybookFilesWithPlaybookDir(t *testing.T) { + testProvisionerProvisionDockerWithPlaybookFiles(t, playbookFilesWithPlaybookDirDockerTemplate) +} + +func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateString string) { if os.Getenv("PACKER_ACC") == "" { t.Skip("This test is only run with PACKER_ACC=1") } @@ -295,7 +306,7 @@ func TestProvisionerProvisionDocker_PlaybookFiles(t *testing.T) { ui := packer.TestUi(t) cache := &packer.FileCache{CacheDir: os.TempDir()} - tpl, err := template.Parse(strings.NewReader(playbookFilesDockerConfig)) + tpl, err := template.Parse(strings.NewReader(templateString)) if err != nil { t.Fatalf("Unable to parse config: %s", err) } @@ -359,20 +370,64 @@ func TestProvisionerProvisionDocker_PlaybookFiles(t *testing.T) { } } +func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { + cmdIndex := 0 + for _, playbook := range playbooks { + playbook = filepath.ToSlash(playbook) + for ; cmdIndex < len(comm.startCommand); cmdIndex++ { + cmd := comm.startCommand[cmdIndex] + if strings.Contains(cmd, "ansible-playbook") && strings.Contains(cmd, playbook) { + break + } + } + if cmdIndex == len(comm.startCommand) { + panic(fmt.Sprintf("Playbook %s was not executed", playbook)) + } + } +} + +func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { + fmt.Println(comm.uploadDestination) + uploadIndex := 0 + for _, playbook := range playbooks { + playbook = filepath.ToSlash(playbook) + for ; uploadIndex < len(comm.uploadDestination); uploadIndex++ { + dest := comm.uploadDestination[uploadIndex] + if strings.HasSuffix(dest, playbook) { + break + } + } + if uploadIndex == len(comm.uploadDestination) { + panic(fmt.Sprintf("Playbook %s was not uploaded", playbook)) + } + } +} + +func assertPlaybooksNotUploaded(comm *communicatorMock, playbooks []string) { + for _, playbook := range playbooks { + playbook = filepath.ToSlash(playbook) + for _, destination := range comm.uploadDestination { + if strings.HasSuffix(destination, playbook) { + panic(fmt.Sprintf("Playbook %s was uploaded", playbook)) + } + } + } +} + func testConfig() map[string]interface{} { m := make(map[string]interface{}) return m } -func createTempFile() string { - file, err := ioutil.TempFile("", "") +func createTempFile(dir string) string { + file, err := ioutil.TempFile(dir, "") if err != nil { panic(fmt.Sprintf("err: %s", err)) } return file.Name() } -func createTempFiles(numFiles int) []string { +func createTempFiles(dir string, numFiles int) []string { files := make([]string, 0, numFiles) defer func() { // Cleanup the files if not all were created. @@ -384,7 +439,7 @@ func createTempFiles(numFiles int) []string { }() for i := 0; i < numFiles; i++ { - files = append(files, createTempFile()) + files = append(files, createTempFile(dir)) } return files } @@ -395,7 +450,7 @@ func removeFiles(files ...string) { } } -const playbookFilesDockerConfig = ` +const playbookFilesDockerTemplate = ` { "builders": [ { @@ -421,3 +476,31 @@ const playbookFilesDockerConfig = ` ] } ` + +const playbookFilesWithPlaybookDirDockerTemplate = ` +{ + "builders": [ + { + "type": "docker", + "image": "williamyeh/ansible:centos7", + "discard": true + } + ], + "provisioners": [ + { + "type": "ansible-local", + "playbook_files": [ + "test-fixtures/hello.yml", + "test-fixtures/world.yml" + ], + "playbook_dir": "test-fixtures" + }, + { + "type": "file", + "source": "/tmp/hello_world", + "destination": "hello_world", + "direction": "download" + } + ] +} +` From daca0c2efe8e77986eb06e5ed2ba734bc28ddec5 Mon Sep 17 00:00:00 2001 From: localghost Date: Mon, 10 Jul 2017 22:05:42 +0200 Subject: [PATCH 07/11] Remove accidental dependency to moby's ioutils package. --- provisioner/ansible-local/provisioner_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 461e53539..1b4bbe5e7 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -12,7 +12,6 @@ import ( "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/provisioner/file" "github.com/hashicorp/packer/template" - "github.com/moby/moby/pkg/ioutils" "os/exec" ) @@ -144,7 +143,7 @@ func TestProvisionerProvision_PlaybookFilesWithPlaybookDir(t *testing.T) { var p Provisioner config := testConfig() - playbook_dir, err := ioutils.TempDir("", "") + playbook_dir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("Failed to create playbook_dir: %s", err) } From 81db142c8a29a45479324e2830cbdb08a5508574 Mon Sep 17 00:00:00 2001 From: localghost Date: Thu, 10 May 2018 22:34:15 +0200 Subject: [PATCH 08/11] Unify handling PlaybookFile and PlaybookFiles. --- provisioner/ansible-local/provisioner.go | 18 +----- provisioner/ansible-local/provisioner_test.go | 55 ++++++++++++++++++- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/provisioner/ansible-local/provisioner.go b/provisioner/ansible-local/provisioner.go index a25aca36c..9a5c95daa 100644 --- a/provisioner/ansible-local/provisioner.go +++ b/provisioner/ansible-local/provisioner.go @@ -111,6 +111,8 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true) if err != nil { errs = packer.MultiErrorAppend(errs, err) + } else { + p.playbookFiles = append(p.playbookFiles, p.config.PlaybookFile) } } @@ -197,14 +199,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } } - if p.config.PlaybookFile != "" { - ui.Message("Uploading main Playbook file...") - src := p.config.PlaybookFile - dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) - if err := p.uploadFile(ui, comm, dst, src); err != nil { - return fmt.Errorf("Error uploading main playbook: %s", err) - } - } else if err := p.provisionPlaybookFiles(ui, comm); err != nil { + if err := p.provisionPlaybookFiles(ui, comm); err != nil { return err } @@ -379,13 +374,6 @@ func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) err } } - if p.config.PlaybookFile != "" { - playbookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile))) - if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { - return err - } - } - for _, playbookFile := range p.playbookFiles { playbookFile = filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile)) if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 1b4bbe5e7..57abc0e87 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -117,6 +117,58 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { } } +func TestProvisionerProvision_PlaybookFile(t *testing.T) { + var p Provisioner + config := testConfig() + + playbook := createTempFile("") + defer os.Remove(playbook) + + config["playbook_file"] = playbook + err := p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + comm := &communicatorMock{} + if err := p.Provision(&uiStub{}, comm); err != nil { + t.Fatalf("err: %s", err) + } + + assertPlaybooksUploaded(comm, []string{playbook}) + assertPlaybooksExecuted(comm, []string{playbook}) +} + +func TestProvisionerProvision_PlaybookFileWithPlaybookDir(t *testing.T) { + var p Provisioner + config := testConfig() + + playbook_dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create playbook_dir: %s", err) + } + defer os.RemoveAll(playbook_dir) + playbook := createTempFile(playbook_dir) + + playbookName := filepath.Base(playbook) + playbookInPlaybookDir := strings.TrimPrefix(playbook, playbook_dir) + + config["playbook_file"] = playbook + config["playbook_dir"] = playbook_dir + err = p.Prepare(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + comm := &communicatorMock{} + if err := p.Provision(&uiStub{}, comm); err != nil { + t.Fatalf("err: %s", err) + } + + assertPlaybooksNotUploaded(comm, []string{playbookName}) + assertPlaybooksExecuted(comm, []string{playbookInPlaybookDir}) +} + func TestProvisionerProvision_PlaybookFiles(t *testing.T) { var p Provisioner config := testConfig() @@ -337,7 +389,6 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin if err != nil { t.Fatalf("Error preparing download: %s", err) } - defer os.Remove("hello_world") // Add hooks so the provisioners run during the build hooks := map[string][]packer.Hook{} @@ -357,6 +408,7 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin t.Fatalf("Error running build %s", err) } defer artifact.Destroy() + defer os.Remove("hello_world") actualContent, err := ioutil.ReadFile("hello_world") if err != nil { @@ -386,7 +438,6 @@ func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { } func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { - fmt.Println(comm.uploadDestination) uploadIndex := 0 for _, playbook := range playbooks { playbook = filepath.ToSlash(playbook) From 6c7aa724eb1da5c740957a5bea02541a0f298022 Mon Sep 17 00:00:00 2001 From: localghost Date: Wed, 16 May 2018 22:43:30 +0200 Subject: [PATCH 09/11] Fix tests after merging with master. --- provisioner/ansible-local/provisioner_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 6eddd2736..4370b060c 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -102,6 +102,7 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { t.Fatal("should have error") } + p = Provisioner{} config["playbook_file"] = playbook_file.Name() config["playbook_files"] = []string{} err = p.Prepare(config) @@ -417,11 +418,10 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin hooks := map[string][]packer.Hook{} hooks[packer.HookProvision] = []packer.Hook{ &packer.ProvisionHook{ - Provisioners: []packer.Provisioner{ - ansible, - download, + Provisioners: []*packer.HookedProvisioner{ + {ansible, nil, ""}, + {download, nil, ""}, }, - ProvisionerTypes: []string{tpl.Provisioners[0].Type, tpl.Provisioners[1].Type}, }, } hook := &packer.DispatchHook{Mapping: hooks} From 46a1c5d9451fbc269d255e7b201c7f86fc77c1c2 Mon Sep 17 00:00:00 2001 From: localghost Date: Sat, 26 May 2018 21:58:08 +0200 Subject: [PATCH 10/11] Revert "Unify handling PlaybookFile and PlaybookFiles." This reverts commit 81db142c8a29a45479324e2830cbdb08a5508574. --- provisioner/ansible-local/provisioner.go | 18 +++++- provisioner/ansible-local/provisioner_test.go | 55 +------------------ 2 files changed, 17 insertions(+), 56 deletions(-) diff --git a/provisioner/ansible-local/provisioner.go b/provisioner/ansible-local/provisioner.go index 098f67ef1..bd4559388 100644 --- a/provisioner/ansible-local/provisioner.go +++ b/provisioner/ansible-local/provisioner.go @@ -114,8 +114,6 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { err = validateFileConfig(p.config.PlaybookFile, "playbook_file", true) if err != nil { errs = packer.MultiErrorAppend(errs, err) - } else { - p.playbookFiles = append(p.playbookFiles, p.config.PlaybookFile) } } @@ -202,7 +200,14 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } } - if err := p.provisionPlaybookFiles(ui, comm); err != nil { + if p.config.PlaybookFile != "" { + ui.Message("Uploading main Playbook file...") + src := p.config.PlaybookFile + dst := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(src))) + if err := p.uploadFile(ui, comm, dst, src); err != nil { + return fmt.Errorf("Error uploading main playbook: %s", err) + } + } else if err := p.provisionPlaybookFiles(ui, comm); err != nil { return err } @@ -384,6 +389,13 @@ func (p *Provisioner) executeAnsible(ui packer.Ui, comm packer.Communicator) err } } + if p.config.PlaybookFile != "" { + playbookFile := filepath.ToSlash(filepath.Join(p.config.StagingDir, filepath.Base(p.config.PlaybookFile))) + if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { + return err + } + } + for _, playbookFile := range p.playbookFiles { playbookFile = filepath.ToSlash(filepath.Join(p.config.StagingDir, playbookFile)) if err := p.executeAnsiblePlaybook(ui, comm, playbookFile, extraArgs, inventory); err != nil { diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 4370b060c..2f6174f3e 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -118,58 +118,6 @@ func TestProvisionerPrepare_PlaybookFiles(t *testing.T) { } } -func TestProvisionerProvision_PlaybookFile(t *testing.T) { - var p Provisioner - config := testConfig() - - playbook := createTempFile("") - defer os.Remove(playbook) - - config["playbook_file"] = playbook - err := p.Prepare(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - comm := &communicatorMock{} - if err := p.Provision(&uiStub{}, comm); err != nil { - t.Fatalf("err: %s", err) - } - - assertPlaybooksUploaded(comm, []string{playbook}) - assertPlaybooksExecuted(comm, []string{playbook}) -} - -func TestProvisionerProvision_PlaybookFileWithPlaybookDir(t *testing.T) { - var p Provisioner - config := testConfig() - - playbook_dir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Failed to create playbook_dir: %s", err) - } - defer os.RemoveAll(playbook_dir) - playbook := createTempFile(playbook_dir) - - playbookName := filepath.Base(playbook) - playbookInPlaybookDir := strings.TrimPrefix(playbook, playbook_dir) - - config["playbook_file"] = playbook - config["playbook_dir"] = playbook_dir - err = p.Prepare(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - comm := &communicatorMock{} - if err := p.Provision(&uiStub{}, comm); err != nil { - t.Fatalf("err: %s", err) - } - - assertPlaybooksNotUploaded(comm, []string{playbookName}) - assertPlaybooksExecuted(comm, []string{playbookInPlaybookDir}) -} - func TestProvisionerProvision_PlaybookFiles(t *testing.T) { var p Provisioner config := testConfig() @@ -413,6 +361,7 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin if err != nil { t.Fatalf("Error preparing download: %s", err) } + defer os.Remove("hello_world") // Add hooks so the provisioners run during the build hooks := map[string][]packer.Hook{} @@ -431,7 +380,6 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin t.Fatalf("Error running build %s", err) } defer artifact.Destroy() - defer os.Remove("hello_world") actualContent, err := ioutil.ReadFile("hello_world") if err != nil { @@ -461,6 +409,7 @@ func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { } func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { + fmt.Println(comm.uploadDestination) uploadIndex := 0 for _, playbook := range playbooks { playbook = filepath.ToSlash(playbook) From 263a3c6910a943cd159d41318a89b544e036852b Mon Sep 17 00:00:00 2001 From: localghost Date: Sat, 26 May 2018 21:59:43 +0200 Subject: [PATCH 11/11] Include some of the refactoring reverted by previous revert commit. --- provisioner/ansible-local/provisioner_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index 2f6174f3e..f3dec49cb 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -361,7 +361,6 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin if err != nil { t.Fatalf("Error preparing download: %s", err) } - defer os.Remove("hello_world") // Add hooks so the provisioners run during the build hooks := map[string][]packer.Hook{} @@ -379,6 +378,7 @@ func testProvisionerProvisionDockerWithPlaybookFiles(t *testing.T, templateStrin if err != nil { t.Fatalf("Error running build %s", err) } + defer os.Remove("hello_world") defer artifact.Destroy() actualContent, err := ioutil.ReadFile("hello_world") @@ -409,7 +409,6 @@ func assertPlaybooksExecuted(comm *communicatorMock, playbooks []string) { } func assertPlaybooksUploaded(comm *communicatorMock, playbooks []string) { - fmt.Println(comm.uploadDestination) uploadIndex := 0 for _, playbook := range playbooks { playbook = filepath.ToSlash(playbook)