From 8fdb4f77e09eba301713f70d132ca3b7c8c0b3c2 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 18 Jun 2015 00:47:33 -0700 Subject: [PATCH] WIP 2/4 tests passing, still need to re-implement ZIP and bare compression files and do some cleanup --- post-processor/compress/post-processor.go | 318 ++++++++---------- .../compress/post-processor_test.go | 140 ++++++++ .../post-processors/compress.html.markdown | 39 ++- 3 files changed, 321 insertions(+), 176 deletions(-) diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index b08465a66..42cea2d35 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -3,19 +3,14 @@ package compress import ( "archive/tar" "archive/zip" - "compress/flate" "compress/gzip" "fmt" "io" "os" "path/filepath" + "regexp" "runtime" - "strings" - "time" - "gopkg.in/yaml.v2" - - "github.com/biogo/hts/bgzf" "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" @@ -24,24 +19,13 @@ import ( "github.com/pierrec/lz4" ) -type Metadata map[string]Metaitem - -type Metaitem struct { - CompSize int64 `yaml:"compsize"` - OrigSize int64 `yaml:"origsize"` - CompType string `yaml:"comptype"` - CompDate string `yaml:"compdate"` -} - type Config struct { common.PackerConfig `mapstructure:",squash"` OutputPath string `mapstructure:"output"` - OutputFile string `mapstructure:"file"` - Compression int `mapstructure:"compression"` - Metadata bool `mapstructure:"metadata"` - NumCPU int `mapstructure:"numcpu"` - Format string `mapstructure:"format"` + Level int `mapstructure:"level"` KeepInputArtifact bool `mapstructure:"keep_input_artifact"` + Archive string + Algorithm string ctx *interpolate.Context } @@ -49,8 +33,52 @@ type PostProcessor struct { config Config } +// ErrInvalidCompressionLevel is returned when the compression level passed to +// gzip is not in the expected range. See compress/flate for details. +var ErrInvalidCompressionLevel = fmt.Errorf( + "Invalid compression level. Expected an integer from -1 to 9.") + +var ErrWrongInputCount = fmt.Errorf( + "Can only have 1 input file when not using tar/zip") + +func detectFromFilename(config *Config) error { + re := regexp.MustCompile("^.+?(?:\\.([a-z0-9]+))?\\.([a-z0-9]+)$") + + extensions := map[string]string{ + "tar": "tar", + "zip": "zip", + "gz": "pgzip", + "lz4": "lz4", + } + + result := re.FindAllString(config.OutputPath, -1) + + // Should we make an archive? E.g. tar or zip? + if result[0] == "tar" { + config.Archive = "tar" + } + if result[1] == "zip" || result[1] == "tar" { + config.Archive = result[1] + // Tar or zip is our final artifact. Bail out. + return nil + } + + // Should we compress the artifact? + algorithm, ok := extensions[result[1]] + if ok { + config.Algorithm = algorithm + // We found our compression algorithm something. Bail out. + return nil + } + + // We didn't find anything. Default to tar + pgzip + config.Algorithm = "pgzip" + config.Archive = "tar" + return fmt.Errorf("Unable to detect compression algorithm") +} + func (p *PostProcessor) Configure(raws ...interface{}) error { - p.config.Compression = -1 + p.config.Level = -1 err := config.Decode(&p.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -73,19 +101,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { "output": &p.config.OutputPath, } - if p.config.Compression > flate.BestCompression { - p.config.Compression = flate.BestCompression - } - if p.config.Compression == -1 { - p.config.Compression = flate.DefaultCompression + if p.config.Level > gzip.BestCompression { + p.config.Level = gzip.BestCompression } - - if p.config.NumCPU < 1 { - p.config.NumCPU = runtime.NumCPU() + if p.config.Level == -1 { + p.config.Level = gzip.DefaultCompression } - runtime.GOMAXPROCS(p.config.NumCPU) - for key, ptr := range templates { if *ptr == "" { errs = packer.MultiErrorAppend( @@ -107,123 +129,113 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } -func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { - // layout shows by example how the reference time should be represented. - const layout = "2006-01-02_15-04-05" - t := time.Now() +func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + + newArtifact := &Artifact{Path: p.config.OutputPath} - if !p.config.Metadata { - return metadata + outputFile, err := os.Create(p.config.OutputPath) + if err != nil { + return nil, false, fmt.Errorf( + "Unable to create archive %s: %s", p.config.OutputPath, err) } - for _, f := range files { - if fi, err := os.Stat(f); err != nil { - continue - } else { - if i, ok := metadata[filepath.Base(f)]; !ok { - metadata[filepath.Base(f)] = Metaitem{CompType: p.config.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} - } else { - i.CompSize = fi.Size() - i.CompDate = t.Format(layout) - metadata[filepath.Base(f)] = i - } + defer outputFile.Close() + + // Setup output interface. If we're using compression, output is a + // compression writer. Otherwise it's just a file. + var output io.WriteCloser + switch p.config.Algorithm { + case "lz4": + lzwriter := lz4.NewWriter(outputFile) + if p.config.Level > gzip.DefaultCompression { + lzwriter.Header.HighCompression = true + } + defer lzwriter.Close() + output = lzwriter + case "pgzip": + output, err = pgzip.NewWriterLevel(outputFile, p.config.Level) + if err != nil { + return nil, false, ErrInvalidCompressionLevel } + defer output.Close() + default: + output = outputFile } - return metadata + + //Archive + switch p.config.Archive { + case "tar": + archiveTar(artifact.Files(), output) + case "zip": + archive := zip.NewWriter(output) + defer archive.Close() + default: + // We have a regular file, so we'll just do an io.Copy + if len(artifact.Files()) != 1 { + return nil, false, fmt.Errorf( + "Can only have 1 input file when not using tar/zip. Found %d "+ + "files: %v", len(artifact.Files()), artifact.Files()) + } + source, err := os.Open(artifact.Files()[0]) + if err != nil { + return nil, false, fmt.Errorf( + "Failed to open source file %s for reading: %s", + artifact.Files()[0], err) + } + defer source.Close() + io.Copy(output, source) + } + + return newArtifact, p.config.KeepInputArtifact, nil } -func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newartifact := &Artifact{Path: p.config.OutputPath} - metafile := filepath.Join(p.config.OutputPath, "metadata") +func archiveTar(files []string, output io.WriteCloser) error { + archive := tar.NewWriter(output) + defer archive.Close() - ui.Say(fmt.Sprintf("[CBEDNARSKI] Creating archive at %s", newartifact.Path)) - _, err := os.Stat(newartifact.Path) - if err == nil { - return nil, false, fmt.Errorf("output dir %s must not exists", newartifact.Path) - } - err = os.MkdirAll(newartifact.Path, 0755) - if err != nil { - return nil, false, fmt.Errorf("failed to create output: %s", err) - } + for _, path := range files { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("Unable to read file %s: %s", path, err) + } + defer file.Close() - p.config.Format += "tar.gzip" - formats := strings.Split(p.config.Format, ".") - ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats length %d", len(formats))) - if len(p.config.Format) == 0 { - ui.Say("[CBEDNARSKI] Formats is empty") - formats[0] = "tar.gzip" - } - files := artifact.Files() - - metadata := make(Metadata, 0) - metadata = p.fillMetadata(metadata, files) - - ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats %#v", formats)) - - for _, compress := range formats { - switch compress { - case "tar": - files, err = p.cmpTAR(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "zip": - files, err = p.cmpZIP(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "pgzip": - files, err = p.cmpPGZIP(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "gzip": - files, err = p.cmpGZIP(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "bgzf": - files, err = p.cmpBGZF(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "lz4": - files, err = p.cmpLZ4(files, p.config.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "e2fs": - files, err = p.cmpE2FS(files, filepath.Join(p.config.OutputPath, p.config.OutputFile)) - metadata = p.fillMetadata(metadata, files) + fi, err := file.Stat() + if err != nil { + return fmt.Errorf("Unable to get fileinfo for %s: %s", path, err) } + + target, err := os.Readlink(path) if err != nil { - return nil, false, fmt.Errorf("Failed to compress: %s", err) + return fmt.Errorf("Failed to readlink for %s: %s", path, err) } - } - if p.config.Metadata { - fp, err := os.Create(metafile) + header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, false, err - } - if buf, err := yaml.Marshal(metadata); err != nil { - fp.Close() - return nil, false, err - } else { - if _, err = fp.Write(buf); err != nil { - fp.Close() - return nil, false, err - } - fp.Close() + return fmt.Errorf("Failed to create tar header for %s: %s", path, err) } - } - newartifact.files = append(newartifact.files, files...) - if p.config.Metadata { - newartifact.files = append(newartifact.files, metafile) - } + if err := archive.WriteHeader(header); err != nil { + return fmt.Errorf("Failed to write tar header for %s: %s", path, err) + } - return newartifact, p.config.KeepInputArtifact, nil + if _, err := io.Copy(archive, file); err != nil { + return fmt.Errorf("Failed to copy %s data to archive: %s", path, err) + } + } + return nil } -func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) +func (p *PostProcessor) cmpTAR(files []string, target string) ([]string, error) { + fw, err := os.Create(target) if err != nil { - return nil, fmt.Errorf("tar error creating tar %s: %s", dst, err) + return nil, fmt.Errorf("tar error creating tar %s: %s", target, err) } defer fw.Close() tw := tar.NewWriter(fw) defer tw.Close() - for _, name := range src { + for _, name := range files { fi, err := os.Stat(name) if err != nil { return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) @@ -250,18 +262,18 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { } fr.Close() } - return []string{dst}, nil + return []string{target}, nil } -func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) { var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) + for _, name := range files { + filename := filepath.Join(target, filepath.Base(name)) fw, err := os.Create(filename) if err != nil { - return nil, fmt.Errorf("gzip error: %s", err) + return nil, fmt.Errorf("gzip error creating archive: %s", err) } - cw, err := gzip.NewWriterLevel(fw, p.config.Compression) + cw, err := gzip.NewWriterLevel(fw, p.config.Level) if err != nil { fw.Close() return nil, fmt.Errorf("gzip error: %s", err) @@ -286,15 +298,16 @@ func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { return res, nil } -func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { +func (p *PostProcessor) cmpPGZIP(files []string, target string) ([]string, error) { var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) + for _, name := range files { + filename := filepath.Join(target, filepath.Base(name)) fw, err := os.Create(filename) if err != nil { return nil, fmt.Errorf("pgzip error: %s", err) } - cw, err := pgzip.NewWriterLevel(fw, p.config.Compression) + cw, err := pgzip.NewWriterLevel(fw, p.config.Level) + cw.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) if err != nil { fw.Close() return nil, fmt.Errorf("pgzip error: %s", err) @@ -332,7 +345,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { fw.Close() return nil, fmt.Errorf("lz4 error: %s", err) } - if p.config.Compression > flate.DefaultCompression { + if p.config.Level > gzip.DefaultCompression { cw.Header.HighCompression = true } fr, err := os.Open(name) @@ -355,43 +368,6 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { return res, nil } -func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - - cw, err := bgzf.NewWriterLevel(fw, p.config.Compression, runtime.NumCPU()) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { - panic("not implemented") -} - func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { fw, err := os.Create(dst) if err != nil { diff --git a/post-processor/compress/post-processor_test.go b/post-processor/compress/post-processor_test.go index 12faeabed..6d28a6698 100644 --- a/post-processor/compress/post-processor_test.go +++ b/post-processor/compress/post-processor_test.go @@ -83,6 +83,111 @@ func TestSimpleCompress(t *testing.T) { } } +func TestZipArchive(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(tarTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.zip") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + +func TestTarArchive(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(tarTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.tar") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + +func TestCompressOptions(t *testing.T) { + if os.Getenv(env.TestEnvVar) == "" { + t.Skip(fmt.Sprintf( + "Acceptance tests skipped unless env '%s' set", env.TestEnvVar)) + } + + ui, artifact, err := setup(t) + if err != nil { + t.Fatalf("Error bootstrapping test: %s", err) + } + if artifact != nil { + defer artifact.Destroy() + } + + tpl, err := template.Parse(strings.NewReader(zipTestCase)) + if err != nil { + t.Fatalf("Unable to parse test config: %s", err) + } + + compressor := PostProcessor{} + compressor.Configure(tpl.PostProcessors[0][0].Config) + artifactOut, _, err := compressor.PostProcess(ui, artifact) + if err != nil { + t.Fatalf("Failed to archive artifact: %s", err) + } + // Cleanup after the test completes + defer artifactOut.Destroy() + + // Verify things look good + _, err = os.Stat("package.gz") + if err != nil { + t.Errorf("Unable to read archive: %s", err) + } +} + const simpleTestCase = ` { "post-processors": [ @@ -93,3 +198,38 @@ const simpleTestCase = ` ] } ` + +const zipTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.zip" + } + ] +} +` + +const tarTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.tar" + } + ] +} +` + +const optionsTestCase = ` +{ + "post-processors": [ + { + "type": "compress", + "output": "package.gz", + "level": 9, + "parallel": false + } + ] +} +` diff --git a/website/source/docs/post-processors/compress.html.markdown b/website/source/docs/post-processors/compress.html.markdown index ea3b9c7ac..6f1430e2e 100644 --- a/website/source/docs/post-processors/compress.html.markdown +++ b/website/source/docs/post-processors/compress.html.markdown @@ -15,17 +15,46 @@ archive. ## Configuration -The configuration for this post-processor is extremely simple. +The minimal required configuration is to specify the output file. This will create a gzipped tarball. -* `output` (string) - The path to save the compressed archive. +* `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file. + + If the extension can't be detected tar+gzip will be used as a fallback. + +If you want more control over how the archive is created you can specify the following settings: + +* `level` (optional, integer) - Specify the compression level, for algorithms that support it. Value from -1 through 9 inclusive. 9 offers the smallest file size, but takes longer +* `keep_input_artifact` (optional, bool) - Keep source files; defaults to false + +## Supported Formats + +Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. ## Example -An example is shown below, showing only the post-processor configuration: +Some minimal examples are shown below, showing only the post-processor configuration: + +```json +{ + "type": "compress", + "output": "archive.tar.gz" +} +``` + +```json +{ + "type": "compress", + "output": "archive.zip" +} +``` + +A more complex example, again showing only the post-processor configuration: -```javascript +```json { "type": "compress", - "output": "foo.tar.gz" + "output": "archive.gz", + "compression": 9, + "parallel": false } ```