From 12cf6650a075feab94e5f6695734c2ab4700d7b0 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 16 Jun 2015 12:10:28 -0700 Subject: [PATCH] Revert compress post-processor to master to get baseline test --- post-processor/compress/artifact.go | 30 +- post-processor/compress/post-processor.go | 403 +++------------------- 2 files changed, 60 insertions(+), 373 deletions(-) diff --git a/post-processor/compress/artifact.go b/post-processor/compress/artifact.go index 054d501d1..34a7ce8d6 100644 --- a/post-processor/compress/artifact.go +++ b/post-processor/compress/artifact.go @@ -8,31 +8,37 @@ import ( const BuilderId = "packer.post-processor.compress" type Artifact struct { - builderId string - dir string - f []string + Path string + Provider string } -func (a *Artifact) BuilderId() string { +func NewArtifact(provider, path string) *Artifact { + return &Artifact{ + Path: path, + Provider: provider, + } +} + +func (*Artifact) BuilderId() string { return BuilderId } -func (a *Artifact) Files() []string { - return a.f +func (self *Artifact) Id() string { + return "" } -func (*Artifact) Id() string { - return "COMPRESS" +func (self *Artifact) Files() []string { + return []string{self.Path} } -func (a *Artifact) String() string { - return fmt.Sprintf("VM compressed files in directory: %s", a.dir) +func (self *Artifact) String() string { + return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path) } func (*Artifact) State(name string) interface{} { return nil } -func (a *Artifact) Destroy() error { - return os.RemoveAll(a.dir) +func (self *Artifact) Destroy() error { + return os.Remove(self.Path) } diff --git a/post-processor/compress/post-processor.go b/post-processor/compress/post-processor.go index 6d28e7c0e..ccf300946 100644 --- a/post-processor/compress/post-processor.go +++ b/post-processor/compress/post-processor.go @@ -2,416 +2,97 @@ package compress import ( "archive/tar" - "archive/zip" - "compress/flate" "compress/gzip" "fmt" "io" "os" - "path/filepath" - "runtime" - "strings" - "time" - "github.com/biogo/hts/bgzf" - "github.com/klauspost/pgzip" "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" - "github.com/pierrec/lz4" - "gopkg.in/yaml.v2" ) -type Metadata map[string]Metaitem - -type Metaitem struct { - CompSize int64 `yaml:"compsize"` - OrigSize int64 `yaml:"origsize"` - CompType string `yaml:"comptype"` - CompDate string `yaml:"compdate"` -} - type Config struct { common.PackerConfig `mapstructure:",squash"` - OutputPath string `mapstructure:"output"` - OutputFile string `mapstructure:"file"` - Compression int `mapstructure:"compression"` - Metadata bool `mapstructure:"metadata"` - NumCPU int `mapstructure:"numcpu"` - Format string `mapstructure:"format"` - KeepInputArtifact bool `mapstructure:"keep_input_artifact"` - ctx *interpolate.Context + OutputPath string `mapstructure:"output"` + + ctx interpolate.Context } type PostProcessor struct { - cfg Config + config Config } -func (p *PostProcessor) Configure(raws ...interface{}) error { - p.cfg.Compression = -1 - err := config.Decode(&p.cfg, &config.DecodeOpts{ +func (self *PostProcessor) Configure(raws ...interface{}) error { + err := config.Decode(&self.config, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ - Exclude: []string{ - // TODO figure out if something needs to go here. - }, + Exclude: []string{}, }, }, raws...) - - errs := new(packer.MultiError) - - if p.cfg.OutputPath == "" { - p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}" - } - - if err = interpolate.Validate(p.cfg.OutputPath, p.cfg.ctx); err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Error parsing target template: %s", err)) - } - - templates := map[string]*string{ - "output": &p.cfg.OutputPath, - } - - if p.cfg.Compression > flate.BestCompression { - p.cfg.Compression = flate.BestCompression - } - if p.cfg.Compression == -1 { - p.cfg.Compression = flate.DefaultCompression - } - - if p.cfg.NumCPU < 1 { - p.cfg.NumCPU = runtime.NumCPU() - } - - runtime.GOMAXPROCS(p.cfg.NumCPU) - - for key, ptr := range templates { - if *ptr == "" { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("%s must be set", key)) - } - - *ptr, err = interpolate.Render(p.cfg.OutputPath, p.cfg.ctx) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Error processing %s: %s", key, err)) - } - } - - if len(errs.Errors) > 0 { - return errs + if err != nil { + return err } return nil } -func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { - // layout shows by example how the reference time should be represented. - const layout = "2006-01-02_15-04-05" - t := time.Now() - - if !p.cfg.Metadata { - return metadata - } - for _, f := range files { - if fi, err := os.Stat(f); err != nil { - continue - } else { - if i, ok := metadata[filepath.Base(f)]; !ok { - metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} - } else { - i.CompSize = fi.Size() - i.CompDate = t.Format(layout) - metadata[filepath.Base(f)] = i - } - } - } - return metadata -} - -func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { - newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath} - metafile := filepath.Join(p.cfg.OutputPath, "metadata") - - _, err := os.Stat(newartifact.dir) - if err == nil { - return nil, false, fmt.Errorf("output dir must not exists: %s", err) - } - err = os.MkdirAll(newartifact.dir, 0755) - if err != nil { - return nil, false, fmt.Errorf("failed to create output: %s", err) - } - - formats := strings.Split(p.cfg.Format, ".") - files := artifact.Files() - - metadata := make(Metadata, 0) - metadata = p.fillMetadata(metadata, files) - - for _, compress := range formats { - switch compress { - case "tar": - files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "zip": - files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - case "pgzip": - files, err = p.cmpPGZIP(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "gzip": - files, err = p.cmpGZIP(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "bgzf": - files, err = p.cmpBGZF(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "lz4": - files, err = p.cmpLZ4(files, p.cfg.OutputPath) - metadata = p.fillMetadata(metadata, files) - case "e2fs": - files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile)) - metadata = p.fillMetadata(metadata, files) - } - if err != nil { - return nil, false, fmt.Errorf("Failed to compress: %s", err) - } - } - - if p.cfg.Metadata { - fp, err := os.Create(metafile) - if err != nil { - return nil, false, err - } - if buf, err := yaml.Marshal(metadata); err != nil { - fp.Close() - return nil, false, err - } else { - if _, err = fp.Write(buf); err != nil { - fp.Close() - return nil, false, err - } - fp.Close() - } - } - - newartifact.f = append(newartifact.f, files...) - if p.cfg.Metadata { - newartifact.f = append(newartifact.f, metafile) - } - - return newartifact, p.cfg.KeepInputArtifact, nil -} +func (self *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + ui.Say(fmt.Sprintf("Creating archive for '%s'", artifact.BuilderId())) -func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) + // Create the compressed archive file at the appropriate OutputPath. + fw, err := os.Create(self.config.OutputPath) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed creating file for compressed archive: %s", self.config.OutputPath) } defer fw.Close() - tw := tar.NewWriter(fw) - defer tw.Close() + gw := gzip.NewWriter(fw) + defer gw.Close() - for _, name := range src { - fi, err := os.Stat(name) + // Iterate through all of the artifact's files and put them into the + // compressed archive using the tar/gzip writers. + for _, path := range artifact.Files() { + fi, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed stating file: %s", path) } - target, _ := os.Readlink(name) + target, _ := os.Readlink(path) header, err := tar.FileInfoHeader(fi, target) if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + return nil, false, fmt.Errorf( + "Failed creating archive header: %s", path) } - if err = tw.WriteHeader(header); err != nil { - return nil, fmt.Errorf("tar error: %s", err) - } + tw := tar.NewWriter(gw) + defer tw.Close() - fr, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("tar error: %s", err) + // Write the header first to the archive. This takes partial data + // from the FileInfo that is grabbed by running the stat command. + if err := tw.WriteHeader(header); err != nil { + return nil, false, fmt.Errorf( + "Failed writing archive header: %s", path) } - if _, err = io.Copy(tw, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("tar error: %s", err) - } - fr.Close() - } - return []string{dst}, nil -} - -func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("gzip error: %s", err) - } - cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression) - if err != nil { - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("gzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression) + // Open the target file for archiving and compressing. + fr, err := os.Open(path) if err != nil { - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) + return nil, false, fmt.Errorf( + "Failed opening file '%s' to write compressed archive.", path) } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("pgzip error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} + defer fr.Close() -func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw := lz4.NewWriter(fw) - if err != nil { - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if p.cfg.Compression > flate.DefaultCompression { - cw.Header.HighCompression = true - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("lz4 error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) { - var res []string - for _, name := range src { - filename := filepath.Join(dst, filepath.Base(name)) - fw, err := os.Create(filename) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - - cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU()) - if err != nil { - return nil, fmt.Errorf("bgzf error: %s", err) - } - fr, err := os.Open(name) - if err != nil { - cw.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - if _, err = io.Copy(cw, fr); err != nil { - cw.Close() - fr.Close() - fw.Close() - return nil, fmt.Errorf("bgzf error: %s", err) - } - cw.Close() - fr.Close() - fw.Close() - res = append(res, filename) - } - return res, nil -} - -func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) { - panic("not implemented") -} - -func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { - fw, err := os.Create(dst) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - defer fw.Close() - - zw := zip.NewWriter(fw) - defer zw.Close() - - for _, name := range src { - header, err := zw.Create(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - fr, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("zip error: %s", err) - } - - if _, err = io.Copy(header, fr); err != nil { - fr.Close() - return nil, fmt.Errorf("zip error: %s", err) + if _, err = io.Copy(tw, fr); err != nil { + return nil, false, fmt.Errorf( + "Failed copying file to archive: %s", path) } - fr.Close() } - return []string{dst}, nil + return NewArtifact(artifact.BuilderId(), self.config.OutputPath), false, nil }