From 8bdb7232642fcb343c982b44e1eb9f8f0c497105 Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Wed, 1 Jan 2014 20:29:53 -0800 Subject: [PATCH 01/37] Do some forward porting of the old work of mitchellh/packer's docker branch. #774 --- config.go | 3 +- plugin/post-processor-docker/main.go | 15 ++++++ plugin/post-processor-docker/main_test.go | 1 + post-processor/docker/post-processor.go | 56 +++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 plugin/post-processor-docker/main.go create mode 100644 plugin/post-processor-docker/main_test.go create mode 100644 post-processor/docker/post-processor.go diff --git a/config.go b/config.go index a2fe49297..0056dae2e 100644 --- a/config.go +++ b/config.go @@ -42,7 +42,8 @@ const defaultConfig = ` "post-processors": { "vagrant": "packer-post-processor-vagrant", - "vsphere": "packer-post-processor-vsphere" + "vsphere": "packer-post-processor-vsphere", + "docker": "packer-post-processor-docker" }, "provisioners": { diff --git a/plugin/post-processor-docker/main.go b/plugin/post-processor-docker/main.go new file mode 100644 index 000000000..1e83f47f4 --- /dev/null +++ b/plugin/post-processor-docker/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/packer/plugin" + "github.com/mitchellh/packer/post-processor/docker" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterPostProcessor(new(docker.PostProcessor)) + server.Serve() +} diff --git a/plugin/post-processor-docker/main_test.go b/plugin/post-processor-docker/main_test.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/plugin/post-processor-docker/main_test.go @@ -0,0 +1 @@ +package main diff --git a/post-processor/docker/post-processor.go b/post-processor/docker/post-processor.go new file mode 100644 index 000000000..a1698b238 --- /dev/null +++ b/post-processor/docker/post-processor.go @@ -0,0 +1,56 @@ +package docker + +import ( + "bytes" + "errors" + "github.com/mitchellh/mapstructure" + "github.com/mitchellh/packer/packer" + "os/exec" +) + +type Config struct { + Registry string + Username string + Password string + Email string +} + +type PostProcessor struct { + config Config +} + +func (p *PostProcessor) Configure(raw ...interface{}) error { + if err := mapstructure.Decode(raw, &p.config); err != nil { + return err + } + + if p.config.Registry == "" { + p.config.Registry = "registry.docker.io" + } + + if p.config.Username == "" { + return errors.New("Username is required to push docker image") + } + + if p.config.Password == "" { + return errors.New("Password is required to push docker image") + } + + return nil +} +func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + id := artifact.Id() + ui.Say("Pushing imgage: " + id) + + // TODO: docker login + + stdout := new(bytes.Buffer) + cmd := exec.Command("docker", "push", id) + cmd.Stdout = stdout + if err := cmd.Run(); err != nil { + ui.Say("Failed to push image: " + id) + return nil, true, err + } + + return nil, true, nil +} From a0e533db41344cda12e2270bdb52502eb6b15edb Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Wed, 1 Jan 2014 22:30:28 -0800 Subject: [PATCH 02/37] Rename docker post processor to docker-push. Implement login to a docker registry, error handling --- config.go | 2 +- .../main.go | 4 +-- .../main_test.go | 0 .../{docker => docker-push}/post-processor.go | 33 +++++++++++++++---- 4 files changed, 29 insertions(+), 10 deletions(-) rename plugin/{post-processor-docker => post-processor-docker-push}/main.go (58%) rename plugin/{post-processor-docker => post-processor-docker-push}/main_test.go (100%) rename post-processor/{docker => docker-push}/post-processor.go (59%) diff --git a/config.go b/config.go index 0056dae2e..7ebdcef66 100644 --- a/config.go +++ b/config.go @@ -43,7 +43,7 @@ const defaultConfig = ` "post-processors": { "vagrant": "packer-post-processor-vagrant", "vsphere": "packer-post-processor-vsphere", - "docker": "packer-post-processor-docker" + "docker-push": "packer-post-processor-docker-push" }, "provisioners": { diff --git a/plugin/post-processor-docker/main.go b/plugin/post-processor-docker-push/main.go similarity index 58% rename from plugin/post-processor-docker/main.go rename to plugin/post-processor-docker-push/main.go index 1e83f47f4..eb45b13bd 100644 --- a/plugin/post-processor-docker/main.go +++ b/plugin/post-processor-docker-push/main.go @@ -2,7 +2,7 @@ package main import ( "github.com/mitchellh/packer/packer/plugin" - "github.com/mitchellh/packer/post-processor/docker" + "github.com/mitchellh/packer/post-processor/docker-push" ) func main() { @@ -10,6 +10,6 @@ func main() { if err != nil { panic(err) } - server.RegisterPostProcessor(new(docker.PostProcessor)) + server.RegisterPostProcessor(new(dockerpush.PostProcessor)) server.Serve() } diff --git a/plugin/post-processor-docker/main_test.go b/plugin/post-processor-docker-push/main_test.go similarity index 100% rename from plugin/post-processor-docker/main_test.go rename to plugin/post-processor-docker-push/main_test.go diff --git a/post-processor/docker/post-processor.go b/post-processor/docker-push/post-processor.go similarity index 59% rename from post-processor/docker/post-processor.go rename to post-processor/docker-push/post-processor.go index a1698b238..64f893e6c 100644 --- a/post-processor/docker/post-processor.go +++ b/post-processor/docker-push/post-processor.go @@ -1,7 +1,6 @@ -package docker +package dockerpush import ( - "bytes" "errors" "github.com/mitchellh/mapstructure" "github.com/mitchellh/packer/packer" @@ -38,18 +37,38 @@ func (p *PostProcessor) Configure(raw ...interface{}) error { return nil } + func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { id := artifact.Id() - ui.Say("Pushing imgage: " + id) + ui.Say("Pushing image: " + id) + + if p.config.Email == "" { + cmd := exec.Command("docker", "login", + "-u=\""+p.config.Username+"\"", + "-p=\""+p.config.Password+"\"") + + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } - // TODO: docker login + } else { + cmd := exec.Command("docker", + "login", + "-u=\""+p.config.Username+"\"", + "-p=\""+p.config.Password+"\"", + "-e=\""+p.config.Email+"\"") - stdout := new(bytes.Buffer) + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } + + } cmd := exec.Command("docker", "push", id) - cmd.Stdout = stdout if err := cmd.Run(); err != nil { ui.Say("Failed to push image: " + id) - return nil, true, err + return nil, false, err } return nil, true, nil From 0ec18a723a3931995cf410d723c653d701299f4d Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Wed, 1 Jan 2014 23:29:27 -0800 Subject: [PATCH 03/37] Finish up parameter parsing and validation. Login to a docker index now works, ready for implementation of the actual push logic. --- post-processor/docker-push/post-processor.go | 120 ++++++++++++++----- 1 file changed, 89 insertions(+), 31 deletions(-) diff --git a/post-processor/docker-push/post-processor.go b/post-processor/docker-push/post-processor.go index 64f893e6c..80fc3b2f5 100644 --- a/post-processor/docker-push/post-processor.go +++ b/post-processor/docker-push/post-processor.go @@ -1,70 +1,128 @@ package dockerpush import ( - "errors" - "github.com/mitchellh/mapstructure" + "fmt" + "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" "os/exec" ) type Config struct { - Registry string - Username string - Password string - Email string + common.PackerConfig `mapstructure:",squash"` + + Registry string `mapstructure:"registry"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Email string `mapstructure:"email"` + + tpl *packer.ConfigTemplate } type PostProcessor struct { config Config } -func (p *PostProcessor) Configure(raw ...interface{}) error { - if err := mapstructure.Decode(raw, &p.config); err != nil { +func (p *PostProcessor) Configure(raws ...interface{}) error { + _, err := common.DecodeConfig(&p.config, raws...) + if err != nil { return err } - if p.config.Registry == "" { - p.config.Registry = "registry.docker.io" + p.config.tpl, err = packer.NewConfigTemplate() + if err != nil { + return err } + p.config.tpl.UserVars = p.config.PackerUserVars + + // Accumulate any errors + errs := new(packer.MultiError) - if p.config.Username == "" { - return errors.New("Username is required to push docker image") + templates := map[string]*string{ + "username": &p.config.Username, + "password": &p.config.Password, } - if p.config.Password == "" { - return errors.New("Password is required to push docker image") + for key, ptr := range templates { + if *ptr == "" { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("%s must be set", key)) + } + + *ptr, err = p.config.tpl.Process(*ptr, nil) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error processing %s: %s", key, err)) + } + } + + if len(errs.Errors) > 0 { + return errs } return nil + } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { id := artifact.Id() ui.Say("Pushing image: " + id) - if p.config.Email == "" { - cmd := exec.Command("docker", "login", - "-u=\""+p.config.Username+"\"", - "-p=\""+p.config.Password+"\"") + if p.config.Registry == "" { + + if p.config.Email == "" { + cmd := exec.Command("docker", + "login", + "-u="+p.config.Username, + "-p="+p.config.Password) + + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } + + } else { + cmd := exec.Command("docker", + "login", + "-u="+p.config.Username, + "-p="+p.config.Password, + "-e="+p.config.Email) + + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err } } else { - cmd := exec.Command("docker", - "login", - "-u=\""+p.config.Username+"\"", - "-p=\""+p.config.Password+"\"", - "-e=\""+p.config.Email+"\"") - - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err - } + if p.config.Email == "" { + cmd := exec.Command("docker", + "login", + "-u="+p.config.Username, + "-p="+p.config.Password, + p.config.Registry) + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } + + } else { + cmd := exec.Command("docker", + "login", + "-u="+p.config.Username, + "-p="+p.config.Password, + "-e="+p.config.Email, + p.config.Registry) + + if err := cmd.Run(); err != nil { + ui.Say("Login to the registry " + p.config.Registry + " failed") + return nil, false, err + } + + } } + cmd := exec.Command("docker", "push", id) if err := cmd.Run(); err != nil { ui.Say("Failed to push image: " + id) From 3d60bfb312d4b638b62a97617bce74ade87ded42 Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Thu, 2 Jan 2014 14:49:14 -0800 Subject: [PATCH 04/37] Add docker-import post-processor. Implemented initial working version of Docker image importing code. #774 --- config.go | 3 +- plugin/post-processor-docker-import/main.go | 15 ++ .../post-processor-docker-import/main_test.go | 1 + .../docker-import/post-processor.go | 155 ++++++++++++++++++ post-processor/docker-push/post-processor.go | 2 +- 5 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 plugin/post-processor-docker-import/main.go create mode 100644 plugin/post-processor-docker-import/main_test.go create mode 100644 post-processor/docker-import/post-processor.go diff --git a/config.go b/config.go index 7ebdcef66..72ed19618 100644 --- a/config.go +++ b/config.go @@ -43,7 +43,8 @@ const defaultConfig = ` "post-processors": { "vagrant": "packer-post-processor-vagrant", "vsphere": "packer-post-processor-vsphere", - "docker-push": "packer-post-processor-docker-push" + "docker-push": "packer-post-processor-docker-push", + "docker-import": "packer-post-processor-docker-import" }, "provisioners": { diff --git a/plugin/post-processor-docker-import/main.go b/plugin/post-processor-docker-import/main.go new file mode 100644 index 000000000..e9446b113 --- /dev/null +++ b/plugin/post-processor-docker-import/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/packer/plugin" + "github.com/mitchellh/packer/post-processor/docker-import" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterPostProcessor(new(dockerimport.PostProcessor)) + server.Serve() +} diff --git a/plugin/post-processor-docker-import/main_test.go b/plugin/post-processor-docker-import/main_test.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/plugin/post-processor-docker-import/main_test.go @@ -0,0 +1 @@ +package main diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go new file mode 100644 index 000000000..ef2ec3e39 --- /dev/null +++ b/post-processor/docker-import/post-processor.go @@ -0,0 +1,155 @@ +package dockerimport + +import ( + "fmt" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/packer" + "io" + "os" + "os/exec" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + Repository string `mapstructure:"repository"` + Tag string `mapstructure:"tag"` + Dockerfile string `mapstructure:"dockerfile"` + + tpl *packer.ConfigTemplate +} + +type PostProcessor struct { + config Config +} + +func (p *PostProcessor) Configure(raws ...interface{}) error { + _, err := common.DecodeConfig(&p.config, raws...) + if err != nil { + return err + } + + p.config.tpl, err = packer.NewConfigTemplate() + if err != nil { + return err + } + p.config.tpl.UserVars = p.config.PackerUserVars + + // Accumulate any errors + errs := new(packer.MultiError) + + templates := map[string]*string{ + "repository": &p.config.Repository, + } + + for key, ptr := range templates { + if *ptr == "" { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("%s must be set", key)) + } + + *ptr, err = p.config.tpl.Process(*ptr, nil) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error processing %s: %s", key, err)) + } + } + + if len(errs.Errors) > 0 { + return errs + } + + return nil + +} + +func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + id := artifact.Id() + ui.Say("Importing image: " + id) + + if p.config.Tag == "" { + + cmd := exec.Command("docker", + "import", + "-", + p.config.Repository) + + stdin, err := cmd.StdinPipe() + + if err != nil { + return nil, false, err + } + + // There should be only one artifact of the Docker builder + file, err := os.Open(artifact.Files()[0]) + + if err != nil { + return nil, false, err + } + + defer file.Close() + + if err := cmd.Start(); err != nil { + ui.Say("Image import failed") + return nil, false, err + } + + go func() { + io.Copy(stdin, file) + // close stdin so that program will exit + stdin.Close() + }() + + cmd.Wait() + + } else { + + cmd := exec.Command("docker", + "import", + "-", + p.config.Repository+":"+p.config.Tag) + + stdin, err := cmd.StdinPipe() + + if err != nil { + return nil, false, err + } + + // There should be only one artifact of the Docker builder + file, err := os.Open(artifact.Files()[0]) + + if err != nil { + return nil, false, err + } + + defer file.Close() + + if err := cmd.Start(); err != nil { + ui.Say("Image import failed") + return nil, false, err + } + + go func() { + io.Copy(stdin, file) + // close stdin so that program will exit + stdin.Close() + }() + + cmd.Wait() + + } + + // Process Dockerfile if provided + if p.config.Dockerfile != "" { + + cmd := exec.Command("docker", "build", id) + if err := cmd.Run(); err != nil { + ui.Say("Failed to build image: " + id) + return nil, false, err + } + + // TODO implement dockerfile provisioning + + } + return nil, false, nil +} diff --git a/post-processor/docker-push/post-processor.go b/post-processor/docker-push/post-processor.go index 80fc3b2f5..0ec2ab492 100644 --- a/post-processor/docker-push/post-processor.go +++ b/post-processor/docker-push/post-processor.go @@ -129,5 +129,5 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac return nil, false, err } - return nil, true, nil + return nil, false, nil } From 7022b620177d07db2a33e3aa58075a9ab1f5a3ef Mon Sep 17 00:00:00 2001 From: Jack Pearkes Date: Mon, 6 Jan 2014 16:13:22 -0500 Subject: [PATCH 05/37] website: update defaults for DO ID's --- website/source/docs/builders/digitalocean.html.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index a2525a840..cf637daff 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -36,14 +36,14 @@ Required: Optional: * `image_id` (int) - The ID of the base image to use. This is the image that - will be used to launch a new droplet and provision it. Defaults to "284203", - which happens to be "Ubuntu 12.04 x64 Server." + will be used to launch a new droplet and provision it. Defaults to "1505447", + which happens to be "Ubuntu 12.04.3 x64 Server." * `region_id` (int) - The ID of the region to launch the droplet in. Consequently, this is the region where the snapshot will be available. This defaults to - "1", which is "New York." + "1", which is "New York 1." -* `size_id` (int) - The ID of the droplet size to use. This defaults to "66," +* `size_id` (int) - The ID of the droplet size to use. This defaults to "66", which is the 512MB droplet. * `private_networking` (bool) - Set to `true` to enable private networking From 8147ad66ec71793263f03b39a26c7940ea7ce1aa Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Mon, 6 Jan 2014 13:42:32 -0800 Subject: [PATCH 06/37] Start adding dockerfile provisioning to docker-import post-processor #774 --- .../docker-import/post-processor.go | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index ef2ec3e39..c2f628af1 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -143,12 +143,35 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac if p.config.Dockerfile != "" { cmd := exec.Command("docker", "build", id) - if err := cmd.Run(); err != nil { + + stdin, err := cmd.StdinPipe() + + if err != nil { + return nil, false, err + } + + // open Dockerfile + file, err := os.Open(p.config.Dockerfile) + + if err != nil { + ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) + return nil, false, err + } + + defer file.Close() + + if err := cmd.Start(); err != nil { ui.Say("Failed to build image: " + id) return nil, false, err } - // TODO implement dockerfile provisioning + go func() { + io.Copy(stdin, file) + // close stdin so that program will exit + stdin.Close() + }() + + cmd.Wait() } return nil, false, nil From 208b330b843a19c9fb8119733a462199f8555eba Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Mon, 6 Jan 2014 13:43:30 -0800 Subject: [PATCH 07/37] docker-import post-processor, add TODO #774 --- post-processor/docker-import/post-processor.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index c2f628af1..bc23f8f8a 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -173,6 +173,9 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac cmd.Wait() + // TODO make sure we re-tag instead of create new image + // automatically use previous image as base of new image + } return nil, false, nil } From 00d3ee42e58dade88c63111d5c6173bd2692082b Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Mon, 6 Jan 2014 15:12:08 -0800 Subject: [PATCH 08/37] docker-import: finish up Dockerfile provisioning, Add TODO for next section #774 --- .../docker-import/post-processor.go | 87 ++++++++++++++----- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index bc23f8f8a..a51e6ba6a 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -67,6 +67,8 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac id := artifact.Id() ui.Say("Importing image: " + id) + // TODO Set artifact ID so that docker-push can use it + if p.config.Tag == "" { cmd := exec.Command("docker", @@ -142,39 +144,76 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac // Process Dockerfile if provided if p.config.Dockerfile != "" { - cmd := exec.Command("docker", "build", id) + if p.config.Tag != "" { - stdin, err := cmd.StdinPipe() + cmd := exec.Command("docker", "build", "-t="+p.config.Repository+":"+p.config.Tag, "-") - if err != nil { - return nil, false, err - } + stdin, err := cmd.StdinPipe() - // open Dockerfile - file, err := os.Open(p.config.Dockerfile) + if err != nil { + return nil, false, err + } - if err != nil { - ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) - return nil, false, err - } + // open Dockerfile + file, err := os.Open(p.config.Dockerfile) - defer file.Close() + if err != nil { + ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) + return nil, false, err + } - if err := cmd.Start(); err != nil { - ui.Say("Failed to build image: " + id) - return nil, false, err - } + ui.Say(id) - go func() { - io.Copy(stdin, file) - // close stdin so that program will exit - stdin.Close() - }() + defer file.Close() - cmd.Wait() + if err := cmd.Start(); err != nil { + ui.Say("Failed to build image: " + id) + return nil, false, err + } + + go func() { + io.Copy(stdin, file) + // close stdin so that program will exit + stdin.Close() + }() + + cmd.Wait() + + } else { + + cmd := exec.Command("docker", "build", "-t="+p.config.Repository, "-") + + stdin, err := cmd.StdinPipe() + + if err != nil { + return nil, false, err + } - // TODO make sure we re-tag instead of create new image - // automatically use previous image as base of new image + // open Dockerfile + file, err := os.Open(p.config.Dockerfile) + + if err != nil { + ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) + return nil, false, err + } + + ui.Say(id) + + defer file.Close() + + if err := cmd.Start(); err != nil { + ui.Say("Failed to build image: " + id) + return nil, false, err + } + + go func() { + io.Copy(stdin, file) + // close stdin so that program will exit + stdin.Close() + }() + + cmd.Wait() + } } return nil, false, nil From 9e5c0f6c6a58a605586934f506021a569b9f9306 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Thu, 9 Jan 2014 08:41:34 -0800 Subject: [PATCH 09/37] HTTPDownloader uses UserAgent from DownloadConfig --- common/download.go | 17 ++++++-- common/download_test.go | 87 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/common/download.go b/common/download.go index 2a0b37ee7..b5798b76c 100644 --- a/common/download.go +++ b/common/download.go @@ -43,6 +43,10 @@ type DownloadConfig struct { // for the downloader will be used to verify with this checksum after // it is downloaded. Checksum []byte + + // What to use for the user agent for HTTP requests. If set to "", use the + // default user agent provided by Go. + UserAgent string } // A DownloadClient helps download, verify checksums, etc. @@ -73,8 +77,8 @@ func HashForType(t string) hash.Hash { func NewDownloadClient(c *DownloadConfig) *DownloadClient { if c.DownloaderMap == nil { c.DownloaderMap = map[string]Downloader{ - "http": new(HTTPDownloader), - "https": new(HTTPDownloader), + "http": &HTTPDownloader{userAgent: c.UserAgent}, + "https": &HTTPDownloader{userAgent: c.UserAgent}, } } @@ -182,8 +186,9 @@ func (d *DownloadClient) VerifyChecksum(path string) (bool, error) { // HTTPDownloader is an implementation of Downloader that downloads // files over HTTP. type HTTPDownloader struct { - progress uint - total uint + progress uint + total uint + userAgent string } func (*HTTPDownloader) Cancel() { @@ -197,6 +202,10 @@ func (d *HTTPDownloader) Download(dst io.Writer, src *url.URL) error { return err } + if d.userAgent != "" { + req.Header.Set("User-Agent", d.userAgent) + } + httpClient := &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, diff --git a/common/download_test.go b/common/download_test.go index f1fec941e..57b4ba7bc 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -4,6 +4,8 @@ import ( "crypto/md5" "encoding/hex" "io/ioutil" + "net/http" + "net/http/httptest" "os" "testing" ) @@ -41,6 +43,91 @@ func TestDownloadClient_VerifyChecksum(t *testing.T) { } } +func TestDownloadClientUsesDefaultUserAgent(t *testing.T) { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("tempfile error: %s", err) + } + defer os.Remove(tf.Name()) + + defaultUserAgent := "" + asserted := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if defaultUserAgent == "" { + defaultUserAgent = r.UserAgent() + } else { + incomingUserAgent := r.UserAgent() + if incomingUserAgent != defaultUserAgent { + t.Fatalf("Expected user agent %s, got: %s", defaultUserAgent, incomingUserAgent) + } + + asserted = true + } + })) + + req, err := http.NewRequest("GET", server.URL, nil) + if err != nil { + t.Fatal(err) + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + } + + _, err = httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + + config := &DownloadConfig{ + Url: server.URL, + TargetPath: tf.Name(), + } + + client := NewDownloadClient(config) + _, err = client.Get() + if err != nil { + t.Fatal(err) + } + + if !asserted { + t.Fatal("User-Agent never observed") + } +} + +func TestDownloadClientSetsUserAgent(t *testing.T) { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("tempfile error: %s", err) + } + defer os.Remove(tf.Name()) + + asserted := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + asserted = true + if r.UserAgent() != "fancy user agent" { + t.Fatalf("Expected useragent fancy user agent, got: %s", r.UserAgent()) + } + })) + config := &DownloadConfig{ + Url: server.URL, + TargetPath: tf.Name(), + UserAgent: "fancy user agent", + } + + client := NewDownloadClient(config) + _, err = client.Get() + if err != nil { + t.Fatal(err) + } + + if !asserted { + t.Fatal("HTTP request never made") + } +} + func TestHashForType(t *testing.T) { if h := HashForType("md5"); h == nil { t.Fatalf("md5 hash is nil") From 454849511679216beedeba2063c24881966a7ea9 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Thu, 9 Jan 2014 13:20:30 -0800 Subject: [PATCH 10/37] StepDownload uses packer version as user agent --- common/step_download.go | 1 + packer/version.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/common/step_download.go b/common/step_download.go index 4bf44eb5a..34156d52d 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -70,6 +70,7 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { CopyFile: false, Hash: HashForType(s.ChecksumType), Checksum: checksum, + UserAgent: packer.VersionString(), } path, err, retry := s.download(config, state) diff --git a/packer/version.go b/packer/version.go index 73c9f3018..dea8eb822 100644 --- a/packer/version.go +++ b/packer/version.go @@ -31,6 +31,15 @@ func (versionCommand) Run(env Environment, args []string) int { env.Ui().Machine("version-prelease", VersionPrerelease) env.Ui().Machine("version-commit", GitCommit) + env.Ui().Say(VersionString()) + return 0 +} + +func (versionCommand) Synopsis() string { + return "print Packer version" +} + +func VersionString() string { var versionString bytes.Buffer fmt.Fprintf(&versionString, "Packer v%s", Version) if VersionPrerelease != "" { @@ -41,10 +50,5 @@ func (versionCommand) Run(env Environment, args []string) int { } } - env.Ui().Say(versionString.String()) - return 0 -} - -func (versionCommand) Synopsis() string { - return "print Packer version" + return versionString.String() } From 5d75944ef300b714f1ba78b4ab9728013e7fab2b Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Fri, 10 Jan 2014 16:35:07 +0100 Subject: [PATCH 11/37] Fixes #806 Build fails if GOPATH has multiple paths --- scripts/devcompile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/devcompile.sh b/scripts/devcompile.sh index 51f1b18f0..ffce3f44f 100755 --- a/scripts/devcompile.sh +++ b/scripts/devcompile.sh @@ -56,4 +56,5 @@ export XC_OS=$(go env GOOS) ./scripts/compile.sh # Move all the compiled things to the PATH -cp pkg/${XC_OS}_${XC_ARCH}/* ${GOPATH}/bin +IFS=: MAIN_GOPATH=( $GOPATH ) +cp pkg/${XC_OS}_${XC_ARCH}/* ${MAIN_GOPATH}/bin From e4cfcb2a8ada6ad82aaf3f784a9651d8cb7697d1 Mon Sep 17 00:00:00 2001 From: Myles Steinhauser Date: Fri, 10 Jan 2014 12:40:43 -0500 Subject: [PATCH 12/37] Add more special scancodes --- builder/qemu/step_type_boot_command.go | 11 +++++++++++ builder/virtualbox/iso/step_type_boot_command.go | 10 ++++++++++ builder/vmware/iso/step_type_boot_command.go | 11 +++++++++++ 3 files changed, 32 insertions(+) diff --git a/builder/qemu/step_type_boot_command.go b/builder/qemu/step_type_boot_command.go index 3b2dd62ad..9bf195669 100644 --- a/builder/qemu/step_type_boot_command.go +++ b/builder/qemu/step_type_boot_command.go @@ -92,6 +92,7 @@ func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction func (*stepTypeBootCommand) Cleanup(multistep.StateBag) {} func vncSendString(c *vnc.ClientConn, original string) { + // Scancodes reference: https://github.com/qemu/qemu/blob/master/ui/vnc_keysym.h special := make(map[string]uint32) special[""] = 0xFF08 special[""] = 0xFFFF @@ -111,6 +112,16 @@ func vncSendString(c *vnc.ClientConn, original string) { special[""] = 0xFFC9 special[""] = 0xFF0D special[""] = 0xFF09 + special[""] = 0xFF52 + special[""] = 0xFF54 + special[""] = 0xFF51 + special[""] = 0xFF53 + special[""] = 0x020 + special[""] = 0xFF63 + special[""] = 0xFF50 + special[""] = 0xFF57 + special[""] = 0xFF55 + special[""] = 0xFF56 shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" diff --git a/builder/virtualbox/iso/step_type_boot_command.go b/builder/virtualbox/iso/step_type_boot_command.go index d479b84e0..e6e06b4ac 100644 --- a/builder/virtualbox/iso/step_type_boot_command.go +++ b/builder/virtualbox/iso/step_type_boot_command.go @@ -118,6 +118,16 @@ func scancodes(message string) []string { special[""] = []string{"44", "c4"} special[""] = []string{"1c", "9c"} special[""] = []string{"0f", "8f"} + special[""] = []string{"48", "c8"} + special[""] = []string{"50", "d0"} + special[""] = []string{"4b", "cb"} + special[""] = []string{"4d", "cd"} + special[""] = []string{"39", "b9"} + special[""] = []string{"52", "d2"} + special[""] = []string{"47", "c7"} + special[""] = []string{"4f", "cf"} + special[""] = []string{"49", "c9"} + special[""] = []string{"51", "d1"} shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" diff --git a/builder/vmware/iso/step_type_boot_command.go b/builder/vmware/iso/step_type_boot_command.go index 89e7ac432..581f8003e 100644 --- a/builder/vmware/iso/step_type_boot_command.go +++ b/builder/vmware/iso/step_type_boot_command.go @@ -116,6 +116,7 @@ func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction func (*stepTypeBootCommand) Cleanup(multistep.StateBag) {} func vncSendString(c *vnc.ClientConn, original string) { + // Scancodes reference: https://github.com/qemu/qemu/blob/master/ui/vnc_keysym.h special := make(map[string]uint32) special[""] = 0xFF08 special[""] = 0xFFFF @@ -135,6 +136,16 @@ func vncSendString(c *vnc.ClientConn, original string) { special[""] = 0xFFC9 special[""] = 0xFF0D special[""] = 0xFF09 + special[""] = 0xFF52 + special[""] = 0xFF54 + special[""] = 0xFF51 + special[""] = 0xFF53 + special[""] = 0x020 + special[""] = 0xFF63 + special[""] = 0xFF50 + special[""] = 0xFF57 + special[""] = 0xFF55 + special[""] = 0xFF56 shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" From 28286d2821f7528b56afd75130ceedb6eaca6033 Mon Sep 17 00:00:00 2001 From: Myles Steinhauser Date: Mon, 13 Jan 2014 00:03:10 -0500 Subject: [PATCH 13/37] Fix tabs vs spaces --- builder/qemu/step_type_boot_command.go | 24 +++++++++---------- .../virtualbox/iso/step_type_boot_command.go | 24 +++++++++---------- builder/vmware/iso/step_type_boot_command.go | 24 +++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/builder/qemu/step_type_boot_command.go b/builder/qemu/step_type_boot_command.go index 9bf195669..5fdc90ca3 100644 --- a/builder/qemu/step_type_boot_command.go +++ b/builder/qemu/step_type_boot_command.go @@ -16,9 +16,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. @@ -112,16 +112,16 @@ func vncSendString(c *vnc.ClientConn, original string) { special[""] = 0xFFC9 special[""] = 0xFF0D special[""] = 0xFF09 - special[""] = 0xFF52 - special[""] = 0xFF54 - special[""] = 0xFF51 - special[""] = 0xFF53 - special[""] = 0x020 - special[""] = 0xFF63 - special[""] = 0xFF50 - special[""] = 0xFF57 - special[""] = 0xFF55 - special[""] = 0xFF56 + special[""] = 0xFF52 + special[""] = 0xFF54 + special[""] = 0xFF51 + special[""] = 0xFF53 + special[""] = 0x020 + special[""] = 0xFF63 + special[""] = 0xFF50 + special[""] = 0xFF57 + special[""] = 0xFF55 + special[""] = 0xFF56 shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" diff --git a/builder/virtualbox/iso/step_type_boot_command.go b/builder/virtualbox/iso/step_type_boot_command.go index e6e06b4ac..e5f90b36d 100644 --- a/builder/virtualbox/iso/step_type_boot_command.go +++ b/builder/virtualbox/iso/step_type_boot_command.go @@ -15,9 +15,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. @@ -118,16 +118,16 @@ func scancodes(message string) []string { special[""] = []string{"44", "c4"} special[""] = []string{"1c", "9c"} special[""] = []string{"0f", "8f"} - special[""] = []string{"48", "c8"} - special[""] = []string{"50", "d0"} - special[""] = []string{"4b", "cb"} - special[""] = []string{"4d", "cd"} - special[""] = []string{"39", "b9"} - special[""] = []string{"52", "d2"} - special[""] = []string{"47", "c7"} - special[""] = []string{"4f", "cf"} - special[""] = []string{"49", "c9"} - special[""] = []string{"51", "d1"} + special[""] = []string{"48", "c8"} + special[""] = []string{"50", "d0"} + special[""] = []string{"4b", "cb"} + special[""] = []string{"4d", "cd"} + special[""] = []string{"39", "b9"} + special[""] = []string{"52", "d2"} + special[""] = []string{"47", "c7"} + special[""] = []string{"4f", "cf"} + special[""] = []string{"49", "c9"} + special[""] = []string{"51", "d1"} shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" diff --git a/builder/vmware/iso/step_type_boot_command.go b/builder/vmware/iso/step_type_boot_command.go index 581f8003e..d021d5d58 100644 --- a/builder/vmware/iso/step_type_boot_command.go +++ b/builder/vmware/iso/step_type_boot_command.go @@ -18,9 +18,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. @@ -136,16 +136,16 @@ func vncSendString(c *vnc.ClientConn, original string) { special[""] = 0xFFC9 special[""] = 0xFF0D special[""] = 0xFF09 - special[""] = 0xFF52 - special[""] = 0xFF54 - special[""] = 0xFF51 - special[""] = 0xFF53 - special[""] = 0x020 - special[""] = 0xFF63 - special[""] = 0xFF50 - special[""] = 0xFF57 - special[""] = 0xFF55 - special[""] = 0xFF56 + special[""] = 0xFF52 + special[""] = 0xFF54 + special[""] = 0xFF51 + special[""] = 0xFF53 + special[""] = 0x020 + special[""] = 0xFF63 + special[""] = 0xFF50 + special[""] = 0xFF57 + special[""] = 0xFF55 + special[""] = 0xFF56 shiftedChars := "~!@#$%^&*()_+{}|:\"<>?" From 5a26624897e898faaab55f901ef92c7fa277b578 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 13 Jan 2014 11:11:55 -0800 Subject: [PATCH 14/37] Update virtualbox-iso.html.markdown Changed type to virtualbox-iso --- website/source/docs/builders/virtualbox-iso.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index 801b7b5a6..490fbeb65 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -5,7 +5,7 @@ page_title: "VirtualBox Builder (from an ISO)" # VirtualBox Builder (from an ISO) -Type: `virtualbox` +Type: `virtualbox-iso` The VirtualBox builder is able to create [VirtualBox](https://www.virtualbox.org/) virtual machines and export them in the OVF format, starting from an From 358b0078c94de3c40c620bd534aba43a96c8b82b Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Mon, 13 Jan 2014 13:21:34 -0800 Subject: [PATCH 15/37] docker-import + docker-push: Add some beginning tests. #774 --- .../docker-import/post-processor.go | 2 -- .../docker-import/post-processor_test.go | 31 +++++++++++++++++++ .../docker-push/post-processor_test.go | 31 +++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 post-processor/docker-import/post-processor_test.go create mode 100644 post-processor/docker-push/post-processor_test.go diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index a51e6ba6a..e0162a9c4 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -67,8 +67,6 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac id := artifact.Id() ui.Say("Importing image: " + id) - // TODO Set artifact ID so that docker-push can use it - if p.config.Tag == "" { cmd := exec.Command("docker", diff --git a/post-processor/docker-import/post-processor_test.go b/post-processor/docker-import/post-processor_test.go new file mode 100644 index 000000000..43ac0b4ef --- /dev/null +++ b/post-processor/docker-import/post-processor_test.go @@ -0,0 +1,31 @@ +package dockerimport + +import ( + "bytes" + "github.com/mitchellh/packer/packer" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{} +} + +func testPP(t *testing.T) *PostProcessor { + var p PostProcessor + if err := p.Configure(testConfig()); err != nil { + t.Fatalf("err: %s", err) + } + + return &p +} + +func testUi() *packer.BasicUi { + return &packer.BasicUi{ + Reader: new(bytes.Buffer), + Writer: new(bytes.Buffer), + } +} + +func TestPostProcessor_ImplementsPostProcessor(t *testing.T) { + var _ packer.PostProcessor = new(PostProcessor) +} diff --git a/post-processor/docker-push/post-processor_test.go b/post-processor/docker-push/post-processor_test.go new file mode 100644 index 000000000..7631da79d --- /dev/null +++ b/post-processor/docker-push/post-processor_test.go @@ -0,0 +1,31 @@ +package dockerpush + +import ( + "bytes" + "github.com/mitchellh/packer/packer" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{} +} + +func testPP(t *testing.T) *PostProcessor { + var p PostProcessor + if err := p.Configure(testConfig()); err != nil { + t.Fatalf("err: %s", err) + } + + return &p +} + +func testUi() *packer.BasicUi { + return &packer.BasicUi{ + Reader: new(bytes.Buffer), + Writer: new(bytes.Buffer), + } +} + +func TestPostProcessor_ImplementsPostProcessor(t *testing.T) { + var _ packer.PostProcessor = new(PostProcessor) +} From c0174309c113e2465f5626815f6671cf539b270d Mon Sep 17 00:00:00 2001 From: Matthew McKeen Date: Mon, 13 Jan 2014 13:37:09 -0800 Subject: [PATCH 16/37] docker-push: add code to handle seperate registry, push a specific repository/tag #774 --- post-processor/docker-push/post-processor.go | 35 ++++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/post-processor/docker-push/post-processor.go b/post-processor/docker-push/post-processor.go index 0ec2ab492..53b8a17a4 100644 --- a/post-processor/docker-push/post-processor.go +++ b/post-processor/docker-push/post-processor.go @@ -10,10 +10,12 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` - Registry string `mapstructure:"registry"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Email string `mapstructure:"email"` + Repository string `mapstructure:"repository"` + Tag string `mapstructure:"tag"` + Registry string `mapstructure:"registry"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Email string `mapstructure:"email"` tpl *packer.ConfigTemplate } @@ -38,8 +40,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs := new(packer.MultiError) templates := map[string]*string{ - "username": &p.config.Username, - "password": &p.config.Password, + "username": &p.config.Username, + "password": &p.config.Password, + "repository": &p.config.Repository, } for key, ptr := range templates { @@ -123,10 +126,22 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac } } - cmd := exec.Command("docker", "push", id) - if err := cmd.Run(); err != nil { - ui.Say("Failed to push image: " + id) - return nil, false, err + if p.config.Tag != "" { + + cmd := exec.Command("docker", "push", p.config.Repository+":"+p.config.Tag) + if err := cmd.Run(); err != nil { + ui.Say("Failed to push image: " + id) + return nil, false, err + } + + } else { + + cmd := exec.Command("docker", "push", p.config.Repository) + if err := cmd.Run(); err != nil { + ui.Say("Failed to push image: " + id) + return nil, false, err + } + } return nil, false, nil From a76650f9ea22433f8291a8c8a2287a0ab0ca52f9 Mon Sep 17 00:00:00 2001 From: Klynton Jessup Date: Tue, 14 Jan 2014 20:13:43 -0700 Subject: [PATCH 17/37] Update post-processor.html.markdown Fixed some typos in the interface{} mentions. --- website/source/docs/extend/post-processor.html.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/extend/post-processor.html.markdown b/website/source/docs/extend/post-processor.html.markdown index bec75b06b..b453dda29 100644 --- a/website/source/docs/extend/post-processor.html.markdown +++ b/website/source/docs/extend/post-processor.html.markdown @@ -46,11 +46,11 @@ type PostProcessor interface { The `Configure` method for each post-processor is called early in the build process to configure the post-processor. The configuration is passed -in as a raw `interface{]`. The configure method is responsible for translating +in as a raw `interface{}`. The configure method is responsible for translating this configuration into an internal structure, validating it, and returning any errors. -For decoding the `interface{]` into a meaningful structure, the +For decoding the `interface{}` into a meaningful structure, the [mapstructure](https://github.com/mitchellh/mapstructure) library is recommended. Mapstructure will take an `interface{}` and decode it into an arbitrarily complex struct. If there are any errors, it generates very From 433c7673fb1b3d317feb69116717a9b4355b5e89 Mon Sep 17 00:00:00 2001 From: Dana Merrick Date: Thu, 16 Jan 2014 12:03:56 -0800 Subject: [PATCH 18/37] Updating the googlecompute builder documentation. --- .../source/docs/builders/googlecompute.markdown | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/website/source/docs/builders/googlecompute.markdown b/website/source/docs/builders/googlecompute.markdown index d68105177..6443a530a 100644 --- a/website/source/docs/builders/googlecompute.markdown +++ b/website/source/docs/builders/googlecompute.markdown @@ -28,16 +28,10 @@ Follow the steps below: 2. Click on the project you want to use Packer with (or create one if you don't have one yet). 3. Click "APIs & auth" in the left sidebar -4. Click "Registered apps" in the left sidebar -5. Click "Register App" and register a "Web Application". Choose any - name you'd like. -7. After creating the app, click "Certificate" (below the OAuth 2.0 Client - ID section), and click "Download JSON". This is your _client secrets JSON_ - file. Make sure you didn't download the JSON from the "OAuth 2.0" section! - This is a common mistake and will cause the builder to not work. -8. Next, click "Generate Certificate". You should be prompted to download - a private key. Note the password for the private key! This private key - is your _client private key_. +4. Click "Credentials" in the left sidebar +5. Click "Create New Client ID" and choose "Service Account" +6. A private key will be downloaded for you. Note the password for the private key! This private key is your _client private key_. +7. After creating the account, click "Download JSON". This is your _client secrets JSON_ file. Make sure you didn't download the JSON from the "OAuth 2.0" section! This is a common mistake and will cause the builder to not work. Finally, one last step, you'll have to convert the `p12` file you got from Google into the PEM format. You can do this with OpenSSL, which From e4d6a157f0c8e52cddbbdc61e6482fb9934cf3dd Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 16 Jan 2014 12:16:12 -0800 Subject: [PATCH 19/37] scripts: Fix slight bash error --- scripts/devcompile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devcompile.sh b/scripts/devcompile.sh index ffce3f44f..f4f26d7e7 100755 --- a/scripts/devcompile.sh +++ b/scripts/devcompile.sh @@ -15,7 +15,7 @@ verify_go () { return 0 fi local IFS=. - local i ver1=($1) ver2=($2) + local i ver1="$1" ver2="$2" for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) do From 602ed10e898d24161dda2a85908eb0542ce44d73 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 16 Jan 2014 15:52:42 -0800 Subject: [PATCH 20/37] scripts: devcompile works properly in Cygwin --- scripts/devcompile.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/devcompile.sh b/scripts/devcompile.sh index f4f26d7e7..4a4274c5b 100755 --- a/scripts/devcompile.sh +++ b/scripts/devcompile.sh @@ -56,5 +56,10 @@ export XC_OS=$(go env GOOS) ./scripts/compile.sh # Move all the compiled things to the PATH +case $(uname) in + CYGWIN*) + GOPATH="$(cygpath $GOPATH)" + ;; +esac IFS=: MAIN_GOPATH=( $GOPATH ) cp pkg/${XC_OS}_${XC_ARCH}/* ${MAIN_GOPATH}/bin From 75f64ef6afedc5559c0a643185ec7bcf02aa48b5 Mon Sep 17 00:00:00 2001 From: jamie brim Date: Fri, 17 Jan 2014 16:49:17 -0800 Subject: [PATCH 21/37] provisioner/ansible: upload playbooks correctly `Provision` calls uploadFile on each path in playbook_paths, but playbook_paths can only contain directories, per bf7530ca28. This changes `Provision` to call uploadDir instead. --- provisioner/ansible-local/provisioner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/ansible-local/provisioner.go b/provisioner/ansible-local/provisioner.go index 1cbbe2626..1347376f0 100644 --- a/provisioner/ansible-local/provisioner.go +++ b/provisioner/ansible-local/provisioner.go @@ -137,7 +137,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } for _, src := range p.config.PlaybookPaths { dst := filepath.Join(p.config.StagingDir, "playbooks", filepath.Base(src)) - if err := p.uploadFile(ui, comm, dst, src); err != nil { + if err := p.uploadDir(ui, comm, dst, src); err != nil { return fmt.Errorf("Error uploading playbooks: %s", err) } } From 8195b8da7f24f75fe52e46ca606e86572a601b79 Mon Sep 17 00:00:00 2001 From: DracoBlue Date: Sat, 18 Jan 2014 22:03:36 +0100 Subject: [PATCH 22/37] Fixed typo in qemu documentation --- website/source/docs/builders/qemu.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/builders/qemu.html.markdown b/website/source/docs/builders/qemu.html.markdown index e432e6796..e6c7f30f0 100644 --- a/website/source/docs/builders/qemu.html.markdown +++ b/website/source/docs/builders/qemu.html.markdown @@ -8,7 +8,7 @@ Type: `qemu` The Qemu builder is able to create [KVM](http://www.linux-kvm.org) and [Xen](http://www.xenproject.org) virtual machine images. Support -for Xen is experimanetal at this time. +for Xen is experimental at this time. The builder builds a virtual machine by creating a new virtual machine from scratch, booting it, installing an OS, rebooting the machine with the From 3857822ef2741eb680b3d635af0f759f69d520a7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 15:19:10 -0800 Subject: [PATCH 23/37] packer: don't crash if arg is empty [GH-832] --- CHANGELOG.md | 1 + packer/environment.go | 2 +- packer/environment_test.go | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcb835f15..be793c167 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ BUG FIXES: +* core: Fix crash case if blank parameters are given to Packer. [GH-832] * builders/docker: user variables work properly. [GH-777] * builder/virtualbox,vmware: iso\_checksum is not required if the checksum type is "none" diff --git a/packer/environment.go b/packer/environment.go index 34dd8ba7d..a7067a682 100644 --- a/packer/environment.go +++ b/packer/environment.go @@ -221,7 +221,7 @@ func (e *coreEnvironment) Cli(args []string) (result int, err error) { // Trim up to the command name for i, v := range args { - if v[0] != '-' { + if len(v) > 0 && v[0] != '-' { args = args[i:] break } diff --git a/packer/environment_test.go b/packer/environment_test.go index b5ac7b9c8..65bc83057 100644 --- a/packer/environment_test.go +++ b/packer/environment_test.go @@ -198,10 +198,17 @@ func TestEnvironment_Cli_CallsRun(t *testing.T) { func TestEnvironment_DefaultCli_Empty(t *testing.T) { defaultEnv := testEnvironment() + // Test with no args exitCode, _ := defaultEnv.Cli([]string{}) if exitCode != 1 { t.Fatalf("bad: %d", exitCode) } + + // Test with only blank args + exitCode, _ = defaultEnv.Cli([]string{""}) + if exitCode != 1 { + t.Fatalf("bad: %d", exitCode) + } } func TestEnvironment_DefaultCli_Help(t *testing.T) { From ffe6e4da33d1e8b4c2b67f70e9eb5440ec972d98 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 15:21:40 -0800 Subject: [PATCH 24/37] Update cHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be793c167..b6764f766 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ BUG FIXES: * builders/docker: user variables work properly. [GH-777] * builder/virtualbox,vmware: iso\_checksum is not required if the checksum type is "none" +* provisioners/ansible-local: Properly upload custom playbooks. [GH-829] ## 0.5.1 (01/02/2014) From d6432f55000b8cad7641e9f8ecefc0506189f7a1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 15:41:33 -0800 Subject: [PATCH 25/37] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6764f766..3bf4eb9af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ BUG FIXES: * builders/docker: user variables work properly. [GH-777] * builder/virtualbox,vmware: iso\_checksum is not required if the checksum type is "none" +* builder/virtualbox,vmware/qemu: Support for additional scancodes for + `boot_command` such as ``, ``, ``, etc. [GH-808] * provisioners/ansible-local: Properly upload custom playbooks. [GH-829] ## 0.5.1 (01/02/2014) From 7ec05423ce9a6cd6aaf8bcaa42d48f5e8fdf707a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 15:48:56 -0800 Subject: [PATCH 26/37] Update CHANGELOG --- CHANGELOG.md | 4 ++++ packer/version.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf4eb9af..c08ffef01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## 0.5.2 (unreleased) +IMPROVEMENTS: + +* core: Most downloads made by Packer now use a custom user agent. [GH-803] + BUG FIXES: * core: Fix crash case if blank parameters are given to Packer. [GH-832] diff --git a/packer/version.go b/packer/version.go index dea8eb822..7fb36c926 100644 --- a/packer/version.go +++ b/packer/version.go @@ -39,6 +39,9 @@ func (versionCommand) Synopsis() string { return "print Packer version" } +// VersionString returns the Packer version in human-readable +// form complete with pre-release and git commit info if it is +// available. func VersionString() string { var versionString bytes.Buffer fmt.Fprintf(&versionString, "Packer v%s", Version) From ebd240789019458801ed9ee6f209d9041c6cf512 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 19:15:25 -0800 Subject: [PATCH 27/37] post-processor/docker-import: refactor /cc @mmckeen - Did a refactor here, CC'd in case you're curious or want to learn :) --- .../docker-import/post-processor.go | 159 ++++-------------- 1 file changed, 37 insertions(+), 122 deletions(-) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index e0162a9c4..c752ad5da 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -12,9 +12,9 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` + Dockerfile string `mapstructure:"dockerfile"` Repository string `mapstructure:"repository"` Tag string `mapstructure:"tag"` - Dockerfile string `mapstructure:"dockerfile"` tpl *packer.ConfigTemplate } @@ -39,7 +39,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs := new(packer.MultiError) templates := map[string]*string{ + "dockerfile": &p.config.Dockerfile, "repository": &p.config.Repository, + "tag": &p.config.Tag, } for key, ptr := range templates { @@ -64,155 +66,68 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - id := artifact.Id() - ui.Say("Importing image: " + id) - - if p.config.Tag == "" { - - cmd := exec.Command("docker", - "import", - "-", - p.config.Repository) - - stdin, err := cmd.StdinPipe() - - if err != nil { - return nil, false, err - } - - // There should be only one artifact of the Docker builder - file, err := os.Open(artifact.Files()[0]) + importRepo := p.config.Repository + if p.config.Tag != "" { + importRepo += ":" + p.config.Tag + } - if err != nil { - return nil, false, err - } + cmd := exec.Command("docker", "import", "-", importRepo) + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, false, err + } - defer file.Close() + // There should be only one artifact of the Docker builder + file, err := os.Open(artifact.Files()[0]) + if err != nil { + return nil, false, err + } + defer file.Close() - if err := cmd.Start(); err != nil { - ui.Say("Image import failed") - return nil, false, err - } + ui.Message("Importing image: " + artifact.Id()) + ui.Message("Repository: " + importRepo) - go func() { - io.Copy(stdin, file) - // close stdin so that program will exit - stdin.Close() - }() + if err := cmd.Start(); err != nil { + return nil, false, err + } - cmd.Wait() + go func() { + defer stdin.Close() + io.Copy(stdin, file) + }() - } else { + cmd.Wait() - cmd := exec.Command("docker", - "import", - "-", - p.config.Repository+":"+p.config.Tag) + // Process Dockerfile if provided + if p.config.Dockerfile != "" { + cmd := exec.Command("docker", "build", "-t="+importRepo, "-") stdin, err := cmd.StdinPipe() - if err != nil { return nil, false, err } - // There should be only one artifact of the Docker builder - file, err := os.Open(artifact.Files()[0]) - + // open Dockerfile + file, err := os.Open(p.config.Dockerfile) if err != nil { + err = fmt.Errorf("Couldn't open Dockerfile: %s", err) return nil, false, err } - defer file.Close() + ui.Message("Running docker build with Dockerfile: " + p.config.Dockerfile) if err := cmd.Start(); err != nil { - ui.Say("Image import failed") + err = fmt.Errorf("Failed to start docker build: %s", err) return nil, false, err } go func() { + defer stdin.Close() io.Copy(stdin, file) - // close stdin so that program will exit - stdin.Close() }() cmd.Wait() - } - - // Process Dockerfile if provided - if p.config.Dockerfile != "" { - - if p.config.Tag != "" { - - cmd := exec.Command("docker", "build", "-t="+p.config.Repository+":"+p.config.Tag, "-") - - stdin, err := cmd.StdinPipe() - - if err != nil { - return nil, false, err - } - - // open Dockerfile - file, err := os.Open(p.config.Dockerfile) - - if err != nil { - ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) - return nil, false, err - } - - ui.Say(id) - - defer file.Close() - - if err := cmd.Start(); err != nil { - ui.Say("Failed to build image: " + id) - return nil, false, err - } - - go func() { - io.Copy(stdin, file) - // close stdin so that program will exit - stdin.Close() - }() - - cmd.Wait() - - } else { - - cmd := exec.Command("docker", "build", "-t="+p.config.Repository, "-") - - stdin, err := cmd.StdinPipe() - - if err != nil { - return nil, false, err - } - - // open Dockerfile - file, err := os.Open(p.config.Dockerfile) - - if err != nil { - ui.Say("Could not open Dockerfile: " + p.config.Dockerfile) - return nil, false, err - } - - ui.Say(id) - - defer file.Close() - - if err := cmd.Start(); err != nil { - ui.Say("Failed to build image: " + id) - return nil, false, err - } - - go func() { - io.Copy(stdin, file) - // close stdin so that program will exit - stdin.Close() - }() - - cmd.Wait() - } - } return nil, false, nil } From 430963f4000ba9bdce738f5686ffd6de612a9679 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 19:30:11 -0800 Subject: [PATCH 28/37] fmt --- builder/qemu/step_type_boot_command.go | 4 ++-- builder/virtualbox/iso/step_type_boot_command.go | 4 ++-- builder/vmware/iso/step_type_boot_command.go | 4 ++-- packer/version.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/qemu/step_type_boot_command.go b/builder/qemu/step_type_boot_command.go index 5fdc90ca3..6416e0ea1 100644 --- a/builder/qemu/step_type_boot_command.go +++ b/builder/qemu/step_type_boot_command.go @@ -16,9 +16,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. diff --git a/builder/virtualbox/iso/step_type_boot_command.go b/builder/virtualbox/iso/step_type_boot_command.go index e5f90b36d..3e087f0df 100644 --- a/builder/virtualbox/iso/step_type_boot_command.go +++ b/builder/virtualbox/iso/step_type_boot_command.go @@ -15,9 +15,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. diff --git a/builder/vmware/iso/step_type_boot_command.go b/builder/vmware/iso/step_type_boot_command.go index d021d5d58..4dbc7f82d 100644 --- a/builder/vmware/iso/step_type_boot_command.go +++ b/builder/vmware/iso/step_type_boot_command.go @@ -18,9 +18,9 @@ import ( const KeyLeftShift uint32 = 0xFFE1 type bootCommandTemplateData struct { - HTTPIP string + HTTPIP string HTTPPort uint - Name string + Name string } // This step "types" the boot command into the VM over VNC. diff --git a/packer/version.go b/packer/version.go index 7fb36c926..31821a807 100644 --- a/packer/version.go +++ b/packer/version.go @@ -53,5 +53,5 @@ func VersionString() string { } } - return versionString.String() + return versionString.String() } From 34dbf72142b48412895d090c8f0b947b134a9eae Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 19:48:06 -0800 Subject: [PATCH 29/37] post-processor/docker-import: have an Artifact /cc @mmckeen --- builder/docker/artifact_import.go | 33 +++++++++++ builder/docker/artifact_import_test.go | 57 +++++++++++++++++++ builder/docker/driver.go | 3 + builder/docker/driver_docker.go | 19 +++++++ builder/docker/driver_mock.go | 10 ++++ .../docker-import/post-processor.go | 54 +++++++----------- 6 files changed, 143 insertions(+), 33 deletions(-) create mode 100644 builder/docker/artifact_import.go create mode 100644 builder/docker/artifact_import_test.go diff --git a/builder/docker/artifact_import.go b/builder/docker/artifact_import.go new file mode 100644 index 000000000..4c926eb53 --- /dev/null +++ b/builder/docker/artifact_import.go @@ -0,0 +1,33 @@ +package docker + +import ( + "fmt" +) + +// ImportArtifact is an Artifact implementation for when a container is +// exported from docker into a single flat file. +type ImportArtifact struct { + BuilderIdValue string + Driver Driver + IdValue string +} + +func (a *ImportArtifact) BuilderId() string { + return a.BuilderIdValue +} + +func (*ImportArtifact) Files() []string { + return nil +} + +func (a *ImportArtifact) Id() string { + return a.IdValue +} + +func (a *ImportArtifact) String() string { + return fmt.Sprintf("Imported Docker image: %s", a.Id()) +} + +func (a *ImportArtifact) Destroy() error { + return a.Driver.DeleteImage(a.Id()) +} diff --git a/builder/docker/artifact_import_test.go b/builder/docker/artifact_import_test.go new file mode 100644 index 000000000..971143953 --- /dev/null +++ b/builder/docker/artifact_import_test.go @@ -0,0 +1,57 @@ +package docker + +import ( + "errors" + "github.com/mitchellh/packer/packer" + "testing" +) + +func TestImportArtifact_impl(t *testing.T) { + var _ packer.Artifact = new(ImportArtifact) +} + +func TestImportArtifactBuilderId(t *testing.T) { + a := &ImportArtifact{BuilderIdValue: "foo"} + if a.BuilderId() != "foo" { + t.Fatalf("bad: %#v", a.BuilderId()) + } +} + +func TestImportArtifactFiles(t *testing.T) { + a := &ImportArtifact{} + if a.Files() != nil { + t.Fatalf("bad: %#v", a.Files()) + } +} + +func TestImportArtifactId(t *testing.T) { + a := &ImportArtifact{IdValue: "foo"} + if a.Id() != "foo" { + t.Fatalf("bad: %#v", a.Id()) + } +} + +func TestImportArtifactDestroy(t *testing.T) { + d := new(MockDriver) + a := &ImportArtifact{ + Driver: d, + IdValue: "foo", + } + + // No error + if err := a.Destroy(); err != nil { + t.Fatalf("err: %s", err) + } + if !d.DeleteImageCalled { + t.Fatal("delete image should be called") + } + if d.DeleteImageId != "foo" { + t.Fatalf("bad: %#v", d.DeleteImageId) + } + + // With an error + d.DeleteImageErr = errors.New("foo") + if err := a.Destroy(); err != d.DeleteImageErr { + t.Fatalf("err: %#v", err) + } +} diff --git a/builder/docker/driver.go b/builder/docker/driver.go index d182ba145..174cc8ef2 100644 --- a/builder/docker/driver.go +++ b/builder/docker/driver.go @@ -8,6 +8,9 @@ import ( // Docker. The Driver interface also allows the steps to be tested since // a mock driver can be shimmed in. type Driver interface { + // Delete an image that is imported into Docker + DeleteImage(id string) error + // Export exports the container with the given ID to the given writer. Export(id string, dst io.Writer) error diff --git a/builder/docker/driver_docker.go b/builder/docker/driver_docker.go index 676045f1a..f55adaba9 100644 --- a/builder/docker/driver_docker.go +++ b/builder/docker/driver_docker.go @@ -15,6 +15,25 @@ type DockerDriver struct { Tpl *packer.ConfigTemplate } +func (d *DockerDriver) DeleteImage(id string) error { + var stderr bytes.Buffer + cmd := exec.Command("docker", "rmi", id) + cmd.Stderr = &stderr + + log.Printf("Deleting image: %s", id) + if err := cmd.Start(); err != nil { + return err + } + + if err := cmd.Wait(); err != nil { + err = fmt.Errorf("Error deleting image: %s\nStderr: %s", + err, stderr.String()) + return err + } + + return nil +} + func (d *DockerDriver) Export(id string, dst io.Writer) error { var stderr bytes.Buffer cmd := exec.Command("docker", "export", id) diff --git a/builder/docker/driver_mock.go b/builder/docker/driver_mock.go index be28d680d..7fa118b28 100644 --- a/builder/docker/driver_mock.go +++ b/builder/docker/driver_mock.go @@ -6,6 +6,10 @@ import ( // MockDriver is a driver implementation that can be used for tests. type MockDriver struct { + DeleteImageCalled bool + DeleteImageId string + DeleteImageErr error + ExportReader io.Reader ExportError error PullError error @@ -25,6 +29,12 @@ type MockDriver struct { VerifyCalled bool } +func (d *MockDriver) DeleteImage(id string) error { + d.DeleteImageCalled = true + d.DeleteImageId = id + return d.DeleteImageErr +} + func (d *MockDriver) Export(id string, dst io.Writer) error { d.ExportCalled = true d.ExportID = id diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index c752ad5da..dcc9f7bac 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -1,18 +1,22 @@ package dockerimport import ( + "bytes" "fmt" + "github.com/mitchellh/packer/builder/docker" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" "io" "os" "os/exec" + "strings" ) +const BuilderId = "packer.post-processor.docker-import" + type Config struct { common.PackerConfig `mapstructure:",squash"` - Dockerfile string `mapstructure:"dockerfile"` Repository string `mapstructure:"repository"` Tag string `mapstructure:"tag"` @@ -39,7 +43,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs := new(packer.MultiError) templates := map[string]*string{ - "dockerfile": &p.config.Dockerfile, "repository": &p.config.Repository, "tag": &p.config.Tag, } @@ -71,7 +74,9 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac importRepo += ":" + p.config.Tag } + var stdout bytes.Buffer cmd := exec.Command("docker", "import", "-", importRepo) + cmd.Stdout = &stdout stdin, err := cmd.StdinPipe() if err != nil { return nil, false, err @@ -96,38 +101,21 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac io.Copy(stdin, file) }() - cmd.Wait() - - // Process Dockerfile if provided - if p.config.Dockerfile != "" { - cmd := exec.Command("docker", "build", "-t="+importRepo, "-") - - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, false, err - } - - // open Dockerfile - file, err := os.Open(p.config.Dockerfile) - if err != nil { - err = fmt.Errorf("Couldn't open Dockerfile: %s", err) - return nil, false, err - } - defer file.Close() - - ui.Message("Running docker build with Dockerfile: " + p.config.Dockerfile) - if err := cmd.Start(); err != nil { - err = fmt.Errorf("Failed to start docker build: %s", err) - return nil, false, err - } - - go func() { - defer stdin.Close() - io.Copy(stdin, file) - }() + if err := cmd.Wait(); err != nil { + err = fmt.Errorf("Error importing container: %s", err) + return nil, false, err + } - cmd.Wait() + id := strings.TrimSpace(stdout.String()) + ui.Message("Imported ID: " + id) + // Build the artifact + driver := &docker.DockerDriver{Tpl: p.config.tpl, Ui: ui} + artifact = &docker.ImportArtifact{ + BuilderIdValue: BuilderId, + Driver: driver, + IdValue: id, } - return nil, false, nil + + return artifact, false, nil } From 8635085665e6cad4fbf7c1de3c2aaed2e33bcfa4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 19:55:01 -0800 Subject: [PATCH 30/37] builder/docker: make Import part of this Driver for reuse --- builder/docker/driver.go | 3 ++ builder/docker/driver_docker.go | 34 ++++++++++++++++++ builder/docker/driver_mock.go | 13 +++++++ .../docker-import/post-processor.go | 36 ++----------------- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/builder/docker/driver.go b/builder/docker/driver.go index 174cc8ef2..ad9bff11f 100644 --- a/builder/docker/driver.go +++ b/builder/docker/driver.go @@ -14,6 +14,9 @@ type Driver interface { // Export exports the container with the given ID to the given writer. Export(id string, dst io.Writer) error + // Import imports a container from a tar file + Import(path, repo string) (string, error) + // Pull should pull down the given image. Pull(image string) error diff --git a/builder/docker/driver_docker.go b/builder/docker/driver_docker.go index f55adaba9..b78fc2a21 100644 --- a/builder/docker/driver_docker.go +++ b/builder/docker/driver_docker.go @@ -6,6 +6,7 @@ import ( "github.com/mitchellh/packer/packer" "io" "log" + "os" "os/exec" "strings" ) @@ -54,6 +55,39 @@ func (d *DockerDriver) Export(id string, dst io.Writer) error { return nil } +func (d *DockerDriver) Import(path string, repo string) (string, error) { + var stdout bytes.Buffer + cmd := exec.Command("docker", "import", "-", repo) + cmd.Stdout = &stdout + stdin, err := cmd.StdinPipe() + if err != nil { + return "", err + } + + // There should be only one artifact of the Docker builder + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + if err := cmd.Start(); err != nil { + return "", err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, file) + }() + + if err := cmd.Wait(); err != nil { + err = fmt.Errorf("Error importing container: %s", err) + return "", err + } + + return strings.TrimSpace(stdout.String()), nil +} + func (d *DockerDriver) Pull(image string) error { cmd := exec.Command("docker", "pull", image) return runAndStream(cmd, d.Ui) diff --git a/builder/docker/driver_mock.go b/builder/docker/driver_mock.go index 7fa118b28..a737c3240 100644 --- a/builder/docker/driver_mock.go +++ b/builder/docker/driver_mock.go @@ -10,6 +10,12 @@ type MockDriver struct { DeleteImageId string DeleteImageErr error + ImportCalled bool + ImportPath string + ImportRepo string + ImportId string + ImportErr error + ExportReader io.Reader ExportError error PullError error @@ -49,6 +55,13 @@ func (d *MockDriver) Export(id string, dst io.Writer) error { return d.ExportError } +func (d *MockDriver) Import(path, repo string) (string, error) { + d.ImportCalled = true + d.ImportPath = path + d.ImportRepo = repo + return d.ImportId, d.ImportErr +} + func (d *MockDriver) Pull(image string) error { d.PullCalled = true d.PullImage = image diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index dcc9f7bac..639e2798e 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -1,15 +1,10 @@ package dockerimport import ( - "bytes" "fmt" "github.com/mitchellh/packer/builder/docker" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" - "io" - "os" - "os/exec" - "strings" ) const BuilderId = "packer.post-processor.docker-import" @@ -74,43 +69,18 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac importRepo += ":" + p.config.Tag } - var stdout bytes.Buffer - cmd := exec.Command("docker", "import", "-", importRepo) - cmd.Stdout = &stdout - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, false, err - } - - // There should be only one artifact of the Docker builder - file, err := os.Open(artifact.Files()[0]) - if err != nil { - return nil, false, err - } - defer file.Close() + driver := &docker.DockerDriver{Tpl: p.config.tpl, Ui: ui} ui.Message("Importing image: " + artifact.Id()) ui.Message("Repository: " + importRepo) - - if err := cmd.Start(); err != nil { - return nil, false, err - } - - go func() { - defer stdin.Close() - io.Copy(stdin, file) - }() - - if err := cmd.Wait(); err != nil { - err = fmt.Errorf("Error importing container: %s", err) + id, err := driver.Import(artifact.Files()[0], importRepo) + if err != nil { return nil, false, err } - id := strings.TrimSpace(stdout.String()) ui.Message("Imported ID: " + id) // Build the artifact - driver := &docker.DockerDriver{Tpl: p.config.tpl, Ui: ui} artifact = &docker.ImportArtifact{ BuilderIdValue: BuilderId, Driver: driver, From 1de226c7482db98a218fb6f0c3e62ca038c2f609 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 19:56:50 -0800 Subject: [PATCH 31/37] post-processor/docker-import: verify proper builder ID --- post-processor/docker-import/post-processor.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index 639e2798e..5571c5e0b 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -64,6 +64,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + if artifact.BuilderId() != docker.BuilderId { + err := fmt.Errorf( + "Unknown artifact type: %s\nCan only import from Docker builder artifacts.", + artifact.BuilderId()) + return nil, false, err + } + importRepo := p.config.Repository if p.config.Tag != "" { importRepo += ":" + p.config.Tag From 4e4a6ffd4b9c9971eb0545fd356e9a108a6c9bea Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 20:20:00 -0800 Subject: [PATCH 32/37] post-processor/docker-import: use the repo tag as the ID --- post-processor/docker-import/post-processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-processor/docker-import/post-processor.go b/post-processor/docker-import/post-processor.go index 5571c5e0b..78543a754 100644 --- a/post-processor/docker-import/post-processor.go +++ b/post-processor/docker-import/post-processor.go @@ -91,7 +91,7 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac artifact = &docker.ImportArtifact{ BuilderIdValue: BuilderId, Driver: driver, - IdValue: id, + IdValue: importRepo, } return artifact, false, nil From c18b74e9cc612ce4e1b008104ccae9c6e350f55e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 20:34:20 -0800 Subject: [PATCH 33/37] post-processor/docker-push --- post-processor/docker-push/post-processor.go | 117 +++---------------- 1 file changed, 18 insertions(+), 99 deletions(-) diff --git a/post-processor/docker-push/post-processor.go b/post-processor/docker-push/post-processor.go index 53b8a17a4..fa3b40f56 100644 --- a/post-processor/docker-push/post-processor.go +++ b/post-processor/docker-push/post-processor.go @@ -2,21 +2,16 @@ package dockerpush import ( "fmt" + "github.com/mitchellh/packer/builder/docker" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/packer" - "os/exec" + "github.com/mitchellh/packer/post-processor/docker-import" + "strings" ) type Config struct { common.PackerConfig `mapstructure:",squash"` - Repository string `mapstructure:"repository"` - Tag string `mapstructure:"tag"` - Registry string `mapstructure:"registry"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Email string `mapstructure:"email"` - tpl *packer.ConfigTemplate } @@ -38,26 +33,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { // Accumulate any errors errs := new(packer.MultiError) - - templates := map[string]*string{ - "username": &p.config.Username, - "password": &p.config.Password, - "repository": &p.config.Repository, - } - - for key, ptr := range templates { - if *ptr == "" { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("%s must be set", key)) - } - - *ptr, err = p.config.tpl.Process(*ptr, nil) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Error processing %s: %s", key, err)) - } - } - if len(errs.Errors) > 0 { return errs } @@ -67,81 +42,25 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - id := artifact.Id() - ui.Say("Pushing image: " + id) - - if p.config.Registry == "" { - - if p.config.Email == "" { - cmd := exec.Command("docker", - "login", - "-u="+p.config.Username, - "-p="+p.config.Password) - - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err - } - - } else { - cmd := exec.Command("docker", - "login", - "-u="+p.config.Username, - "-p="+p.config.Password, - "-e="+p.config.Email) - - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err - } - - } - - } else { - if p.config.Email == "" { - cmd := exec.Command("docker", - "login", - "-u="+p.config.Username, - "-p="+p.config.Password, - p.config.Registry) - - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err - } - - } else { - cmd := exec.Command("docker", - "login", - "-u="+p.config.Username, - "-p="+p.config.Password, - "-e="+p.config.Email, - p.config.Registry) - - if err := cmd.Run(); err != nil { - ui.Say("Login to the registry " + p.config.Registry + " failed") - return nil, false, err - } - - } + if artifact.BuilderId() != dockerimport.BuilderId { + err := fmt.Errorf( + "Unknown artifact type: %s\nCan only import from docker-import artifacts.", + artifact.BuilderId()) + return nil, false, err } - if p.config.Tag != "" { - - cmd := exec.Command("docker", "push", p.config.Repository+":"+p.config.Tag) - if err := cmd.Run(); err != nil { - ui.Say("Failed to push image: " + id) - return nil, false, err - } + driver := &docker.DockerDriver{Tpl: p.config.tpl, Ui: ui} - } else { - - cmd := exec.Command("docker", "push", p.config.Repository) - if err := cmd.Run(); err != nil { - ui.Say("Failed to push image: " + id) - return nil, false, err - } + // Get the name. We strip off any tags from the name because the + // push doesn't use those. + name := artifact.Id() + if i := strings.Index(name, ":"); i >= 0 { + name = name[:i] + } + ui.Message("Pushing: " + name) + if err := driver.Push(name); err != nil { + return nil, false, err } return nil, false, nil From e8768785a06aa205973e9cf311a92a00fbb698b4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 20:42:42 -0800 Subject: [PATCH 34/37] website: add docs for docker-import and docker-push --- builder/docker/driver.go | 3 ++ builder/docker/driver_docker.go | 5 +++ builder/docker/driver_mock.go | 10 +++++ .../source/docs/builders/docker.html.markdown | 33 +++++++++----- .../docker-import.html.markdown | 44 +++++++++++++++++++ .../post-processors/docker-push.html.markdown | 28 ++++++++++++ website/source/layouts/docs.erb | 2 + 7 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 website/source/docs/post-processors/docker-import.html.markdown create mode 100644 website/source/docs/post-processors/docker-push.html.markdown diff --git a/builder/docker/driver.go b/builder/docker/driver.go index ad9bff11f..aaed2fc52 100644 --- a/builder/docker/driver.go +++ b/builder/docker/driver.go @@ -20,6 +20,9 @@ type Driver interface { // Pull should pull down the given image. Pull(image string) error + // Push pushes an image to a Docker index/registry. + Push(name string) error + // StartContainer starts a container and returns the ID for that container, // along with a potential error. StartContainer(*ContainerConfig) (string, error) diff --git a/builder/docker/driver_docker.go b/builder/docker/driver_docker.go index b78fc2a21..5c43e0078 100644 --- a/builder/docker/driver_docker.go +++ b/builder/docker/driver_docker.go @@ -93,6 +93,11 @@ func (d *DockerDriver) Pull(image string) error { return runAndStream(cmd, d.Ui) } +func (d *DockerDriver) Push(name string) error { + cmd := exec.Command("docker", "push", name) + return runAndStream(cmd, d.Ui) +} + func (d *DockerDriver) StartContainer(config *ContainerConfig) (string, error) { // Build up the template data var tplData startContainerTemplate diff --git a/builder/docker/driver_mock.go b/builder/docker/driver_mock.go index a737c3240..a48bb99f8 100644 --- a/builder/docker/driver_mock.go +++ b/builder/docker/driver_mock.go @@ -16,6 +16,10 @@ type MockDriver struct { ImportId string ImportErr error + PushCalled bool + PushName string + PushErr error + ExportReader io.Reader ExportError error PullError error @@ -68,6 +72,12 @@ func (d *MockDriver) Pull(image string) error { return d.PullError } +func (d *MockDriver) Push(name string) error { + d.PushCalled = true + d.PushName = name + return d.PushErr +} + func (d *MockDriver) StartContainer(config *ContainerConfig) (string, error) { d.StartCalled = true d.StartConfig = config diff --git a/website/source/docs/builders/docker.html.markdown b/website/source/docs/builders/docker.html.markdown index fbced5009..af827a14d 100644 --- a/website/source/docs/builders/docker.html.markdown +++ b/website/source/docs/builders/docker.html.markdown @@ -64,15 +64,31 @@ Optional: `["run", "-d", "-i", "-t", "-v", "{{.Volumes}}", "{{.Image}}", "/bin/bash"]`. As you can see, you have a couple template variables to customize, as well. -## Using the generated artifact +## Using the Artifact Once the tar artifact has been generated, you will likely want to import, tag, -and push it to a container repository. Until packer supports management of the -docker image metadata, this process is manual. For example, the following will -import `mycontainer-123456789.tar` to the repository -`registry.mydomain.com/mycontainer`, tagged with `latest`: +and push it to a container repository. Packer can do this for you automatically +with the [docker-import](/docs/post-processors/docker-import.html) and +[docker-push](/docs/post-processors/docker-push.html) post-processors. - sudo docker import - registry.mydomain.com/mycontainer:latest < mycontainer-123456789.tar +The example below shows a full configuration that would import and push +the created image: + +
+{
+    "post-processors": [
+		[
+			{ "type": "docker-import", "repository": "mitchellh/packer", "tag": "0.7" },
+			"docker-push"
+		]
+	]
+}
+
+ +If you want to do this manually, however, perhaps from a script, you can +import the image using the process below: + + docker import - registry.mydomain.com/mycontainer:latest < artifact.tar You can then add additional tags and push the image as usual with `docker tag` and `docker push`, respectively. @@ -103,8 +119,3 @@ by Packer in the future: volumes, and other metadata. Packer builds a raw Docker container image that has none of this metadata. You can pass in much of this metadata at runtime with `docker run`. - -* Images made without dockerfiles are missing critical metadata that - make them easily pushable to the Docker registry. You can work around - this by using a metadata-only Dockerfile with the exported image and - building that. A future Packer version will automatically do this for you. diff --git a/website/source/docs/post-processors/docker-import.html.markdown b/website/source/docs/post-processors/docker-import.html.markdown new file mode 100644 index 000000000..e2ca3c93b --- /dev/null +++ b/website/source/docs/post-processors/docker-import.html.markdown @@ -0,0 +1,44 @@ +--- +layout: "docs" +page_title: "docker-import Post-Processor" +--- + +# Docker Import Post-Processor + +Type: `docker-import` + +The Docker import post-processor takes an artifact from the +[docker builder](/docs/builders/docker.html) and imports it with Docker +locally. This allows you to apply a repository and tag to the image +and lets you use the other Docker post-processors such as +[docker-push](/docs/post-processors/docker-push.html) to push the image +to a registry. + +## Configuration + +The configuration for this post-processor is extremely simple. At least +a repository is required. The tag is optional. + +* `repository` (string) - The repository of the imported image. + +* `tag` (string) - The tag for the imported image. By default this is not + set. + +## Example + +An example is shown below, showing only the post-processor configuration: + +
+{
+  "type": "docker-import",
+  "repository": "mitchellh/packer",
+  "tag": "0.7"
+}
+
+ +This example would take the image created by the Docker builder +and import it into the local Docker process with a name of `mitchellh/packer:0.7`. + +Following this, you can use the +[docker-push](/docs/post-processors/docker-push.html) +post-processor to push it to a registry, if you want. diff --git a/website/source/docs/post-processors/docker-push.html.markdown b/website/source/docs/post-processors/docker-push.html.markdown new file mode 100644 index 000000000..602e126e0 --- /dev/null +++ b/website/source/docs/post-processors/docker-push.html.markdown @@ -0,0 +1,28 @@ +--- +layout: "docs" +page_title: "Docker Push Post-Processor" +--- + +# Docker Push Post-Processor + +Type: `docker-push` + +The Docker push post-processor takes an artifact from the +[docker-import](/docs/post-processors/docker-import.html) post-processor +and pushes it to a Docker registry. + +
+Before you use this, you must manually docker login +to the proper repository. A future version of Packer will automate this +for you, but for now you must manually do this. +
+ +## Configuration + +This post-processor has no configuration! Simply add it to your chain +of post-processors and the image will be uploaded. + +## Example + +For an example of using docker-push, see the section on using +generated artifacts from the [docker builder](/docs/builders/docker.html). diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index d8a70dc56..30744b372 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -54,6 +54,8 @@ From 3e85bc8bf81b078e32bd4d07ae0efba267d3ccb9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 20:48:03 -0800 Subject: [PATCH 35/37] website: update formatting for docker docs --- website/source/docs/builders/docker.html.markdown | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/source/docs/builders/docker.html.markdown b/website/source/docs/builders/docker.html.markdown index af827a14d..315d5ff63 100644 --- a/website/source/docs/builders/docker.html.markdown +++ b/website/source/docs/builders/docker.html.markdown @@ -78,7 +78,11 @@ the created image: { "post-processors": [ [ - { "type": "docker-import", "repository": "mitchellh/packer", "tag": "0.7" }, + { + "type": "docker-import", + "repository": "mitchellh/packer", + "tag": "0.7" + }, "docker-push" ] ] From 87f7ba6cdb8abb6a3cfc4bddaf5340f2110462e7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Jan 2014 20:49:17 -0800 Subject: [PATCH 36/37] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c08ffef01..935b002ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## 0.5.2 (unreleased) +FEATURES: + +* **New post-processor:** `docker-import` - Import a Docker image + and give it a specific repository/tag. +* **New post-processor:** `docker-push` - Push an imported image to + a registry. + IMPROVEMENTS: * core: Most downloads made by Packer now use a custom user agent. [GH-803] From cc4564c05274fd72c023a8916d02f40da4068682 Mon Sep 17 00:00:00 2001 From: yosssi Date: Tue, 21 Jan 2014 01:43:51 +0900 Subject: [PATCH 37/37] Change the word `aws_access_key` to `aws_secret_key`. --- website/source/intro/getting-started/build-image.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index e37f56664..158733bb9 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -63,7 +63,7 @@ briefly. Create a file `example.json` and fill it with the following contents: } -When building, you'll pass in the `aws_access_key` and `aws_access_key` as +When building, you'll pass in the `aws_access_key` and `aws_secret_key` as a [user variable](/docs/templates/user-variables.html), keeping your secret keys out of the template. You can create security credentials on [this page](https://console.aws.amazon.com/iam/home?#security_credential).