From dc3c55cf8e7366514018b4c065a40dc18917bd56 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 11 Aug 2015 22:22:52 -0700 Subject: [PATCH 1/7] Implemented downloader for the docker communicator so we can pull files out of a container --- builder/docker/communicator.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 4fcd9b658..31ccc2579 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -194,8 +194,36 @@ func (c *Communicator) UploadDir(dst string, src string, exclude []string) error return nil } +// Download pulls a file out of a container using `docker cp`. We have a source +// path and want to write to an io.Writer, not a file. We use - to make docker +// cp to write to stdout, and then copy the stream to our destination io.Writer. func (c *Communicator) Download(src string, dst io.Writer) error { - panic("not implemented") + + log.Printf("Downloading file from container: %s:%s", c.ContainerId, src) + localCmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", c.ContainerId, src), "-") + + pipe, err := localCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("Failed to open pipe: %s", err) + } + + err = localCmd.Start() + if err != nil { + return fmt.Errorf("Failed to start download: %s", err) + } + + numBytes, err := io.Copy(dst, pipe) + if err != nil { + return fmt.Errorf("Failed to pipe download: %s", err) + } else { + log.Printf("Copied %d bytes for %s", numBytes, src) + } + + if err = localCmd.Wait(); err != nil { + return fmt.Errorf("Failed to download '%s' from container: %s", src, err) + } + + return nil } // canExec tells us whether `docker exec` is supported From 047382eec9e0cc39e5dbdd9ecd46fb73c7943f91 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 11 Aug 2015 22:30:19 -0700 Subject: [PATCH 2/7] Style tweak --- builder/docker/communicator.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 31ccc2579..38126366c 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -207,17 +207,15 @@ func (c *Communicator) Download(src string, dst io.Writer) error { return fmt.Errorf("Failed to open pipe: %s", err) } - err = localCmd.Start() - if err != nil { + if err = localCmd.Start(); err != nil { return fmt.Errorf("Failed to start download: %s", err) } numBytes, err := io.Copy(dst, pipe) if err != nil { return fmt.Errorf("Failed to pipe download: %s", err) - } else { - log.Printf("Copied %d bytes for %s", numBytes, src) } + log.Printf("Copied %d bytes for %s", numBytes, src) if err = localCmd.Wait(); err != nil { return fmt.Errorf("Failed to download '%s' from container: %s", src, err) From 3523ffdce14b6e27b640b959f405fd972ca8a22e Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 12 Aug 2015 11:36:10 -0700 Subject: [PATCH 3/7] Farewell extra line. You were pretty but out of place. --- builder/docker/communicator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 38126366c..8af54bdfe 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -198,7 +198,6 @@ func (c *Communicator) UploadDir(dst string, src string, exclude []string) error // path and want to write to an io.Writer, not a file. We use - to make docker // cp to write to stdout, and then copy the stream to our destination io.Writer. func (c *Communicator) Download(src string, dst io.Writer) error { - log.Printf("Downloading file from container: %s:%s", c.ContainerId, src) localCmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", c.ContainerId, src), "-") From de9ecd2d62cc0cc38dc66293cd98e3599e6703f5 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 12 Aug 2015 11:36:33 -0700 Subject: [PATCH 4/7] Add a test fixture file --- builder/docker/test-fixtures/cake | 1 + 1 file changed, 1 insertion(+) create mode 100644 builder/docker/test-fixtures/cake diff --git a/builder/docker/test-fixtures/cake b/builder/docker/test-fixtures/cake new file mode 100644 index 000000000..63d40b126 --- /dev/null +++ b/builder/docker/test-fixtures/cake @@ -0,0 +1 @@ +chocolate cake is delicious From 62c5e8358d4045e5ee1ba64956e3536a5952bb4d Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 12 Aug 2015 11:36:56 -0700 Subject: [PATCH 5/7] Added a test for docker upload and download --- builder/docker/communicator_test.go | 117 +++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/builder/docker/communicator_test.go b/builder/docker/communicator_test.go index f75a89d96..221356723 100644 --- a/builder/docker/communicator_test.go +++ b/builder/docker/communicator_test.go @@ -1,10 +1,125 @@ package docker import ( - "github.com/mitchellh/packer/packer" + "crypto/sha256" + "io/ioutil" + "os" + "os/exec" + "runtime" + "strings" "testing" + + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/provisioner/file" + "github.com/mitchellh/packer/template" ) func TestCommunicator_impl(t *testing.T) { var _ packer.Communicator = new(Communicator) } + +func TestUploadDownload(t *testing.T) { + ui := packer.TestUi(t) + cache := &packer.FileCache{CacheDir: os.TempDir()} + + tpl, err := template.Parse(strings.NewReader(dockerBuilderConfig)) + if err != nil { + t.Fatalf("Unable to parse config: %s", err) + } + + // Make sure we only run this on linux hosts + if os.Getenv("PACKER_ACC") == "" { + t.Skip("This test is only run with PACKER_ACC=1") + } + if runtime.GOOS != "linux" { + t.Skip("This test is only supported on linux") + } + cmd := exec.Command("docker", "-v") + cmd.Run() + if !cmd.ProcessState.Success() { + t.Error("docker command not found; please make sure docker is installed") + } + + // Setup the builder + builder := &Builder{} + warnings, err := builder.Prepare(tpl.Builders["docker"].Config) + if err != nil { + t.Fatalf("Error preparing configuration %s", err) + } + if len(warnings) > 0 { + t.Fatal("Encountered configuration warnings; aborting") + } + + // Setup the provisioners + upload := &file.Provisioner{} + err = upload.Prepare(tpl.Provisioners[0].Config) + if err != nil { + t.Fatalf("Error preparing upload: %s", err) + } + download := &file.Provisioner{} + err = download.Prepare(tpl.Provisioners[1].Config) + if err != nil { + t.Fatalf("Error preparing download: %s", err) + } + // Preemptive cleanup + defer os.Remove("delicious-cake") + + // Add hooks so the provisioners run during the build + hooks := map[string][]packer.Hook{} + hooks[packer.HookProvision] = []packer.Hook{ + &packer.ProvisionHook{ + Provisioners: []packer.Provisioner{ + upload, + download, + }, + }, + } + hook := &packer.DispatchHook{Mapping: hooks} + + // Run things + artifact, err := builder.Run(ui, hook, cache) + if err != nil { + t.Fatalf("Error running build %s", err) + } + // Preemptive cleanup + defer artifact.Destroy() + + // Verify that the thing we downloaded is the same thing we sent up. + inputFile, err := ioutil.ReadFile("test-fixtures/cake") + if err != nil { + t.Fatalf("Unable to read input file: %s", err) + } + outputFile, err := ioutil.ReadFile("delicious-cake") + if err != nil { + t.Fatalf("Unable to read output file: %s", err) + } + if sha256.Sum256(inputFile) != sha256.Sum256(outputFile) { + t.Fatalf("Input and output files do not match\nInput:\n%s\nOutput:\n%s\n", inputFile, outputFile) + } +} + +const dockerBuilderConfig = ` +{ + "builders": [ + { + "type": "docker", + "image": "alpine", + "export_path": "alpine.tar", + "run_command": ["-d", "-i", "-t", "{{.Image}}", "/bin/sh"] + } + ], + "provisioners": [ + { + "type": "file", + "source": "test-fixtures/cake", + "destination": "/chocolate-cake" + }, + { + "type": "file", + "source": "/chocolate-cake", + "destination": "delicious-cake", + "direction": "download" + } + ] +} +` From 8cdd07895217f11dc2b8ac34082fc7dc01ed733b Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 12 Aug 2015 11:48:47 -0700 Subject: [PATCH 6/7] Changed fixtures so we can do a directory test too --- builder/docker/test-fixtures/cake | 1 - builder/docker/test-fixtures/manycakes/chocolate | 1 + builder/docker/test-fixtures/manycakes/vanilla | 1 + builder/docker/test-fixtures/onecakes/strawberry | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 builder/docker/test-fixtures/cake create mode 100644 builder/docker/test-fixtures/manycakes/chocolate create mode 100644 builder/docker/test-fixtures/manycakes/vanilla create mode 100644 builder/docker/test-fixtures/onecakes/strawberry diff --git a/builder/docker/test-fixtures/cake b/builder/docker/test-fixtures/cake deleted file mode 100644 index 63d40b126..000000000 --- a/builder/docker/test-fixtures/cake +++ /dev/null @@ -1 +0,0 @@ -chocolate cake is delicious diff --git a/builder/docker/test-fixtures/manycakes/chocolate b/builder/docker/test-fixtures/manycakes/chocolate new file mode 100644 index 000000000..a2286c928 --- /dev/null +++ b/builder/docker/test-fixtures/manycakes/chocolate @@ -0,0 +1 @@ +chocolate! diff --git a/builder/docker/test-fixtures/manycakes/vanilla b/builder/docker/test-fixtures/manycakes/vanilla new file mode 100644 index 000000000..000a45578 --- /dev/null +++ b/builder/docker/test-fixtures/manycakes/vanilla @@ -0,0 +1 @@ +vanilla! diff --git a/builder/docker/test-fixtures/onecakes/strawberry b/builder/docker/test-fixtures/onecakes/strawberry new file mode 100644 index 000000000..b663de3a9 --- /dev/null +++ b/builder/docker/test-fixtures/onecakes/strawberry @@ -0,0 +1 @@ +strawberry! From 5ad4b0e97e186f5414045c1aa42b313fb3e6df65 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Wed, 12 Aug 2015 12:16:26 -0700 Subject: [PATCH 7/7] Added tests and handle the tar format from docker cp - --- builder/docker/communicator.go | 12 +++++++++++- builder/docker/communicator_test.go | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 8af54bdfe..fb88a4491 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -1,6 +1,7 @@ package docker import ( + "archive/tar" "bytes" "fmt" "io" @@ -210,7 +211,16 @@ func (c *Communicator) Download(src string, dst io.Writer) error { return fmt.Errorf("Failed to start download: %s", err) } - numBytes, err := io.Copy(dst, pipe) + // When you use - to send docker cp to stdout it is streamed as a tar; this + // enables it to work with directories. We don't actually support + // directories in Download() but we still need to handle the tar format. + archive := tar.NewReader(pipe) + _, err = archive.Next() + if err != nil { + return fmt.Errorf("Failed to read header from tar stream: %s", err) + } + + numBytes, err := io.Copy(dst, archive) if err != nil { return fmt.Errorf("Failed to pipe download: %s", err) } diff --git a/builder/docker/communicator_test.go b/builder/docker/communicator_test.go index 221356723..db0bfcfe8 100644 --- a/builder/docker/communicator_test.go +++ b/builder/docker/communicator_test.go @@ -61,8 +61,10 @@ func TestUploadDownload(t *testing.T) { if err != nil { t.Fatalf("Error preparing download: %s", err) } - // Preemptive cleanup - defer os.Remove("delicious-cake") + // Preemptive cleanup. Honestly I don't know why you would want to get rid + // of my strawberry cake. It's so tasty! Do you not like cake? Are you a + // cake-hater? Or are you keeping all the cake all for yourself? So selfish! + defer os.Remove("my-strawberry-cake") // Add hooks so the provisioners run during the build hooks := map[string][]packer.Hook{} @@ -85,16 +87,18 @@ func TestUploadDownload(t *testing.T) { defer artifact.Destroy() // Verify that the thing we downloaded is the same thing we sent up. - inputFile, err := ioutil.ReadFile("test-fixtures/cake") + // Complain loudly if it isn't. + inputFile, err := ioutil.ReadFile("test-fixtures/onecakes/strawberry") if err != nil { t.Fatalf("Unable to read input file: %s", err) } - outputFile, err := ioutil.ReadFile("delicious-cake") + outputFile, err := ioutil.ReadFile("my-strawberry-cake") if err != nil { t.Fatalf("Unable to read output file: %s", err) } if sha256.Sum256(inputFile) != sha256.Sum256(outputFile) { - t.Fatalf("Input and output files do not match\nInput:\n%s\nOutput:\n%s\n", inputFile, outputFile) + t.Fatalf("Input and output files do not match\n"+ + "Input:\n%s\nOutput:\n%s\n", inputFile, outputFile) } } @@ -111,13 +115,13 @@ const dockerBuilderConfig = ` "provisioners": [ { "type": "file", - "source": "test-fixtures/cake", - "destination": "/chocolate-cake" + "source": "test-fixtures/onecakes/strawberry", + "destination": "/strawberry-cake" }, { "type": "file", - "source": "/chocolate-cake", - "destination": "delicious-cake", + "source": "/strawberry-cake", + "destination": "my-strawberry-cake", "direction": "download" } ]