From a3f0308e920483343c7947c4cbcff8608af40c9c Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Sat, 5 Mar 2016 01:40:16 -0600 Subject: [PATCH] Re-implemented the support for the floppy_files keyword in order to remain backwards-compatible with templates using the old syntax. Moved the support for recursive paths from the floppy_files keyword to the new floppy_contents keyword. Shifted some of the code around to add better logging of what's actually being copied. Added a couple of unit-tests for the new floppy_contents implementation. Ensured that all files that were being added were also being included in state.FilesAdded so that the older unit-tests will work. --- builder/qemu/builder.go | 4 +- builder/vmware/iso/builder.go | 3 +- builder/vmware/vmx/builder.go | 3 +- common/floppy_config.go | 13 ++- common/step_create_floppy.go | 107 ++++++++++++++++++----- common/step_create_floppy_test.go | 138 ++++++++++++++++++++++++++++++ 6 files changed, 243 insertions(+), 25 deletions(-) diff --git a/builder/qemu/builder.go b/builder/qemu/builder.go index 588313d06..fdddd275b 100644 --- a/builder/qemu/builder.go +++ b/builder/qemu/builder.go @@ -93,7 +93,6 @@ type Config struct { DiskDiscard string `mapstructure:"disk_discard"` SkipCompaction bool `mapstructure:"skip_compaction"` DiskCompression bool `mapstructure:"disk_compression"` - FloppyFiles []string `mapstructure:"floppy_files"` Format string `mapstructure:"format"` Headless bool `mapstructure:"headless"` DiskImage bool `mapstructure:"disk_image"` @@ -365,7 +364,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe steps = append(steps, new(stepPrepareOutputDir), &common.StepCreateFloppy{ - Files: b.config.FloppyFiles, + Files: b.config.FloppyConfig.FloppyFiles, + Contents: b.config.FloppyConfig.FloppyContents, }, new(stepCreateDisk), new(stepCopyDisk), diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index 5a6495cb2..9bd4795e5 100755 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -231,7 +231,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Force: b.config.PackerForce, }, &common.StepCreateFloppy{ - Files: b.config.FloppyFiles, + Files: b.config.FloppyConfig.FloppyFiles, + Contents: b.config.FloppyConfig.FloppyContents, }, &stepRemoteUpload{ Key: "floppy_path", diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go index d82f03f1c..a58661023 100644 --- a/builder/vmware/vmx/builder.go +++ b/builder/vmware/vmx/builder.go @@ -62,7 +62,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Force: b.config.PackerForce, }, &common.StepCreateFloppy{ - Files: b.config.FloppyFiles, + Files: b.config.FloppyConfig.FloppyFiles, + Contents: b.config.FloppyConfig.FloppyContents, }, &StepCloneVMX{ OutputDir: b.config.OutputDir, diff --git a/common/floppy_config.go b/common/floppy_config.go index bd5c5f8c7..3ba515e44 100644 --- a/common/floppy_config.go +++ b/common/floppy_config.go @@ -8,7 +8,8 @@ import ( ) type FloppyConfig struct { - FloppyFiles []string `mapstructure:"floppy_files"` + FloppyFiles []string `mapstructure:"floppy_files"` + FloppyContents []string `mapstructure:"floppy_contents"` } func (c *FloppyConfig) Prepare(ctx *interpolate.Context) []error { @@ -24,5 +25,15 @@ func (c *FloppyConfig) Prepare(ctx *interpolate.Context) []error { } } + if c.FloppyContents == nil { + c.FloppyContents = make([]string, 0) + } + + for _, path := range c.FloppyContents { + if _, err := os.Stat(path); err != nil { + errs = append(errs, fmt.Errorf("Bad Floppy disk directory '%s': %s", path, err)) + } + } + return errs } diff --git a/common/step_create_floppy.go b/common/step_create_floppy.go index f863b8d9f..2564b3ae4 100644 --- a/common/step_create_floppy.go +++ b/common/step_create_floppy.go @@ -19,7 +19,8 @@ import ( // The floppy disk doesn't support sub-directories. Only files at the // root level are supported. type StepCreateFloppy struct { - Files []string + Files []string + Contents []string floppyPath string @@ -27,7 +28,7 @@ type StepCreateFloppy struct { } func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction { - if len(s.Files) == 0 { + if len(s.Files) == 0 && len(s.Contents) == 0 { log.Println("No floppy files specified. Floppy disk will not be made.") return multistep.ActionContinue } @@ -85,17 +86,88 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - // Get the root directory to the filesystem + // Get the root directory to the filesystem and create a cache for any directories within log.Println("Reading the root directory from the filesystem") rootDir, err := fatFs.RootDir() if err != nil { state.Put("error", fmt.Errorf("Error creating floppy: %s", err)) return multistep.ActionHalt } + cache := fsDirectoryCache(rootDir) + + // Utility functions for walking through a directory grabbing all files flatly + globFiles := func(files []string, list chan string) { + for _,filename := range files { + if strings.IndexAny(filename, "*?[") >= 0 { + matches,_ := filepath.Glob(filename) + if err != nil { continue } + + for _,match := range matches { + list <- match + } + continue + } + list <- filename + } + close(list) + } + + var crawlDirectoryFiles []string + crawlDirectory := func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + crawlDirectoryFiles = append(crawlDirectoryFiles, path) + ui.Message(fmt.Sprintf("Adding file: %s", path)) + } + return nil + } + crawlDirectoryFiles = []string{} + + // Collect files and copy them flatly...because floppy_files is broken on purpose. + var filelist chan string + filelist = make(chan string) + go globFiles(s.Files, filelist) + + ui.Message("Copying files flatly from floppy_files") + for { + filename, ok := <-filelist + if !ok { break } + + finfo,err := os.Stat(filename) + if err != nil { + state.Put("error", fmt.Errorf("Error trying to stat : %s : %s", filename, err)) + return multistep.ActionHalt + } + + // walk through directory adding files to the root of the fs + if finfo.IsDir() { + ui.Message(fmt.Sprintf("Copying directory: %s", filename)) + + err := filepath.Walk(filename, crawlDirectory) + if err != nil { + state.Put("error", fmt.Errorf("Error adding file from floppy_files : %s : %s", filename, err)) + return multistep.ActionHalt + } + + for _,crawlfilename := range crawlDirectoryFiles { + s.Add(cache, crawlfilename) + s.FilesAdded[crawlfilename] = true + } + + crawlDirectoryFiles = []string{} + continue + } + + // add just a single file + ui.Message(fmt.Sprintf("Copying file: %s", filename)) + s.Add(cache, filename) + s.FilesAdded[filename] = true + } + ui.Message("Done copying files from floppy_files") // Collect all paths (expanding wildcards) into pathqueue + ui.Message("Collecting paths from floppy_contents") var pathqueue []string - for _,filename := range s.Files { + for _,filename := range s.Contents { if strings.IndexAny(filename, "*?[") >= 0 { matches,err := filepath.Glob(filename) if err != nil { @@ -110,23 +182,18 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction { } pathqueue = append(pathqueue, filename) } + ui.Message(fmt.Sprintf("Resulting paths from floppy_contents : %v", pathqueue)) // Go over each path in pathqueue and copy it. - getDirectory := fsDirectoryCache(rootDir) for _,src := range pathqueue { - ui.Message(fmt.Sprintf("Copying: %s", src)) - err = s.Add(getDirectory, src) + ui.Message(fmt.Sprintf("Recursively copying : %s", src)) + err = s.Add(cache, src) if err != nil { state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", src, err)) return multistep.ActionHalt } - - // FIXME: setting this map according to each pathqueue entry breaks - // our testcases, because it only keeps track of the number of files - // that are set here instead of actually verifying against the - // filesystem...heh -// s.FilesAdded[src] = true } + ui.Message("Done copying paths from floppy_contents") // Set the path to the floppy so it can be used later state.Put("floppy_path", s.floppyPath) @@ -134,7 +201,7 @@ func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionContinue } -func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error { +func (s *StepCreateFloppy) Add(dircache directoryCache, src string) error { finfo,err := os.Stat(src) if err != nil { return fmt.Errorf("Error adding path to floppy: %s", err) @@ -146,7 +213,7 @@ func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error { if err != nil { return err } defer inputF.Close() - d,err := dir("") + d,err := dircache("") if err != nil { return err } entry,err := d.AddFile(path.Base(src)) @@ -167,7 +234,7 @@ func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error { if fi.Mode().IsDir() { base,err := removeBase(basedirectory, pathname) if err != nil { return err } - _,err = dir(filepath.ToSlash(base)) + _,err = dircache(filepath.ToSlash(base)) return err } directory,filename := filepath.Split(pathname) @@ -179,7 +246,7 @@ func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error { if err != nil { return err } defer inputF.Close() - wd,err := dir(filepath.ToSlash(base)) + wd,err := dircache(filepath.ToSlash(base)) if err != nil { return err } entry,err := wd.AddFile(filename) @@ -189,7 +256,7 @@ func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error { if err != nil { return err } _,err = io.Copy(fatFile,inputF) - s.FilesAdded[filename] = true + s.FilesAdded[pathname] = true return err } @@ -233,8 +300,8 @@ func removeBase(base string, path string) (string,error) { // entry associated with a given path. If an fs.Directory entry is not found // then it will be created relative to the rootDirectory argument that is // passed. -type getFsDirectory func(string) (fs.Directory,error) -func fsDirectoryCache(rootDirectory fs.Directory) getFsDirectory { +type directoryCache func(string) (fs.Directory,error) +func fsDirectoryCache(rootDirectory fs.Directory) directoryCache { var cache map[string]fs.Directory cache = make(map[string]fs.Directory) diff --git a/common/step_create_floppy_test.go b/common/step_create_floppy_test.go index d1a3000a7..61dfac68e 100644 --- a/common/step_create_floppy_test.go +++ b/common/step_create_floppy_test.go @@ -7,10 +7,53 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strconv" "testing" + "log" + "strings" + "fmt" ) +// utility function for returning a directory structure as a list of strings +func getDirectory(path string) []string { + var result []string + walk := func(path string, info os.FileInfo, err error) error { + if info.IsDir() && !strings.HasSuffix(path, "/") { + path = path + "/" + } + result = append(result, filepath.ToSlash(path)) + return nil + } + filepath.Walk(path, walk) + return result +} + +// utility function for creating a directory structure +type createFileContents func(string) []byte +func createDirectory(path string, hier []string, fileContents createFileContents) error { + if fileContents == nil { + fileContents = func(string) []byte { + return []byte{} + } + } + for _,filename := range hier { + p := filepath.Join(path, filename) + if strings.HasSuffix(filename, "/") { + err := os.MkdirAll(p, 0) + if err != nil { return err } + continue + } + f,err := os.Create(p) + if err != nil { return err } + _,err = f.Write(fileContents(filename)) + if err != nil { return err } + err = f.Close() + if err != nil { return err } + } + return nil +} + func TestStepCreateFloppy_Impl(t *testing.T) { var raw interface{} raw = new(StepCreateFloppy) @@ -191,3 +234,98 @@ func xxxTestStepCreateFloppy_notfound(t *testing.T) { } } } + +func TestStepCreateFloppyContents(t *testing.T) { + // file-system hierarchies + hierarchies := [][]string{ + []string{"file1", "file2", "file3"}, + []string{"dir1/", "dir1/file1", "dir1/file2", "dir1/file3"}, + []string{"dir1/", "dir1/file1", "dir1/subdir1/", "dir1/subdir1/file1", "dir1/subdir1/file2", "dir2/", "dir2/subdir1/", "dir2/subdir1/file1","dir2/subdir1/file2"}, + } + + type contentsTest struct { + contents []string + result []string + } + + // keep in mind that .FilesAdded doesn't keep track of the target filename or directories, but rather the source filename. + contents := [][]contentsTest{ + []contentsTest{ + contentsTest{contents:[]string{"file1","file2","file3"},result:[]string{"file1","file2","file3"}}, + contentsTest{contents:[]string{"file?"},result:[]string{"file1","file2","file3"}}, + contentsTest{contents:[]string{"*"},result:[]string{"file1","file2","file3"}}, + }, + []contentsTest{ + contentsTest{contents:[]string{"dir1"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}}, + contentsTest{contents:[]string{"dir1/file1","dir1/file2","dir1/file3"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}}, + contentsTest{contents:[]string{"*"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}}, + contentsTest{contents:[]string{"*/*"},result:[]string{"dir1/file1","dir1/file2","dir1/file3"}}, + }, + []contentsTest{ + contentsTest{contents:[]string{"dir1"},result:[]string{"dir1/file1","dir1/subdir1/file1","dir1/subdir1/file2"}}, + contentsTest{contents:[]string{"dir2/*"},result:[]string{"dir2/subdir1/file1","dir2/subdir1/file2"}}, + contentsTest{contents:[]string{"dir2/subdir1"},result:[]string{"dir2/subdir1/file1","dir2/subdir1/file2"}}, + contentsTest{contents:[]string{"dir?"},result:[]string{"dir1/file1","dir1/subdir1/file1","dir1/subdir1/file2","dir2/subdir1/file1","dir2/subdir1/file2"}}, + }, + } + + // create the hierarchy for each file + for i,hier := range hierarchies { + t.Logf("Trying with hierarchy : %v",hier) + + // create the temp directory + dir, err := ioutil.TempDir("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + + // create the file contents + err = createDirectory(dir, hier, nil) + if err != nil { t.Fatalf("err: %s", err) } + t.Logf("Making %v", hier) + + for _,test := range contents[i] { + // createa new state and step + state := testStepCreateFloppyState(t) + step := new(StepCreateFloppy) + + // modify step.Contents with ones from testcase + step.Contents = []string{} + for _,c := range test.contents { + step.Contents = append(step.Contents, filepath.Join(dir,filepath.FromSlash(c))) + } + log.Println(fmt.Sprintf("Trying against floppy_contents : %v",step.Contents)) + + // run the step + if action := step.Run(state); action != multistep.ActionContinue { + t.Fatalf("bad action: %#v for %v : %v", action, step.Contents, state.Get("error")) + } + + if _, ok := state.GetOk("error"); ok { + t.Fatalf("state should be ok for %v : %v", step.Contents, state.Get("error")) + } + + floppy_path := state.Get("floppy_path").(string) + if _, err := os.Stat(floppy_path); err != nil { + t.Fatalf("file not found: %s for %v : %v", floppy_path, step.Contents) + } + + // check the FilesAdded array to see if it matches + for _,rpath := range test.result { + fpath := filepath.Join(dir, filepath.FromSlash(rpath)) + if !step.FilesAdded[fpath] { + t.Fatalf("unable to find file: %s for %v", fpath, step.Contents) + } + } + + // cleanup the step + step.Cleanup(state) + + if _, err := os.Stat(floppy_path); err == nil { + t.Fatalf("file found: %s for %v", floppy_path, step.Contents) + } + } + // remove the temp directory + os.RemoveAll(dir) + } +}