From bb2209004026cd96b3ccc0c7818d0b53d8514eef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 13 Sep 2014 17:45:56 -0700 Subject: [PATCH] config/module: start, lots of initial work --- config/module/folder_storage.go | 65 +++++++++++++++++++++ config/module/folder_storage_test.go | 60 +++++++++++++++++++ config/module/get.go | 53 +++++++++++++++++ config/module/get_file.go | 41 +++++++++++++ config/module/module.go | 7 +++ config/module/module_test.go | 21 +++++++ config/module/storage.go | 13 +++++ config/module/test-fixtures/basic/main.tf | 1 + config/module/tree.go | 71 +++++++++++++++++++++++ 9 files changed, 332 insertions(+) create mode 100644 config/module/folder_storage.go create mode 100644 config/module/folder_storage_test.go create mode 100644 config/module/get.go create mode 100644 config/module/get_file.go create mode 100644 config/module/module.go create mode 100644 config/module/module_test.go create mode 100644 config/module/storage.go create mode 100644 config/module/test-fixtures/basic/main.tf create mode 100644 config/module/tree.go diff --git a/config/module/folder_storage.go b/config/module/folder_storage.go new file mode 100644 index 0000000000..dfb79748af --- /dev/null +++ b/config/module/folder_storage.go @@ -0,0 +1,65 @@ +package module + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "os" + "path/filepath" +) + +// FolderStorage is an implementation of the Storage interface that manages +// modules on the disk. +type FolderStorage struct { + // StorageDir is the directory where the modules will be stored. + StorageDir string +} + +// Dir implements Storage.Dir +func (s *FolderStorage) Dir(source string) (d string, e bool, err error) { + d = s.dir(source) + _, err = os.Stat(d) + if err == nil { + // Directory exists + e = true + return + } + if os.IsNotExist(err) { + // Directory doesn't exist + d = "" + e = false + err = nil + return + } + + // An error + d = "" + e = false + return +} + +// Get implements Storage.Get +func (s *FolderStorage) Get(source string, update bool) error { + dir := s.dir(source) + if !update { + if _, err := os.Stat(dir); err == nil { + // If the directory already exists, then we're done since + // we're not updating. + return nil + } else if !os.IsNotExist(err) { + // If the error we got wasn't a file-not-exist error, then + // something went wrong and we should report it. + return fmt.Errorf("Error reading module directory: %s", err) + } + } + + // Get the source. This always forces an update. + return Get(dir, source) +} + +// dir returns the directory name internally that we'll use to map to +// internally. +func (s *FolderStorage) dir(source string) string { + sum := md5.Sum([]byte(source)) + return filepath.Join(s.StorageDir, hex.EncodeToString(sum[:])) +} diff --git a/config/module/folder_storage_test.go b/config/module/folder_storage_test.go new file mode 100644 index 0000000000..7be9cbe7b4 --- /dev/null +++ b/config/module/folder_storage_test.go @@ -0,0 +1,60 @@ +package module + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestFolderStorage_impl(t *testing.T) { + var _ Storage = new(FolderStorage) +} + +func TestFolderStorage(t *testing.T) { + s := &FolderStorage{StorageDir: tempDir(t)} + + module := testModule("basic") + + // A module shouldn't exist at first... + _, ok, err := s.Dir(module) + if err != nil { + t.Fatalf("err: %s", err) + } + if ok { + t.Fatal("should not exist") + } + + // We can get it + err = s.Get(module, false) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Now the module exists + dir, ok, err := s.Dir(module) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("should exist") + } + + mainPath := filepath.Join(dir, "main.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } + +} + +func tempDir(t *testing.T) string { + dir, err := ioutil.TempDir("", "tf") + if err != nil { + t.Fatalf("err: %s", err) + } + if err := os.RemoveAll(dir); err != nil { + t.Fatalf("err: %s", err) + } + + return dir +} diff --git a/config/module/get.go b/config/module/get.go new file mode 100644 index 0000000000..270ca815a6 --- /dev/null +++ b/config/module/get.go @@ -0,0 +1,53 @@ +package module + +import ( + "fmt" + "net/url" +) + +// Getter defines the interface that schemes must implement to download +// and update modules. +type Getter interface { + // Get downloads the given URL into the given directory. This always + // assumes that we're updating and gets the latest version that it can. + // + // The directory may already exist (if we're updating). If it is in a + // format that isn't understood, an error should be returned. Get shouldn't + // simply nuke the directory. + Get(string, *url.URL) error +} + +// Getters is the mapping of scheme to the Getter implementation that will +// be used to get a dependency. +var Getters map[string]Getter + +func init() { + Getters = map[string]Getter{ + "file": new(FileGetter), + } +} + +// Get downloads the module specified by src into the folder specified by +// dst. If dst already exists, Get will attempt to update it. +// +// src is a URL, whereas dst is always just a file path to a folder. This +// folder doesn't need to exist. It will be created if it doesn't exist. +func Get(dst, src string) error { + u, err := url.Parse(src) + if err != nil { + return err + } + + g, ok := Getters[u.Scheme] + if !ok { + return fmt.Errorf( + "module download not supported for scheme '%s'", u.Scheme) + } + + err = g.Get(dst, u) + if err != nil { + err = fmt.Errorf("error downloading module '%s': %s", src, err) + } + + return err +} diff --git a/config/module/get_file.go b/config/module/get_file.go new file mode 100644 index 0000000000..66ead7e51f --- /dev/null +++ b/config/module/get_file.go @@ -0,0 +1,41 @@ +package module + +import ( + "fmt" + "net/url" + "os" + "path/filepath" +) + +// FileGetter is a Getter implementation that will download a module from +// a file scheme. +type FileGetter struct{} + +func (g *FileGetter) Get(dst string, u *url.URL) error { + fi, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + + // If the destination already exists, it must be a symlink + if err == nil { + mode := fi.Mode() + if mode&os.ModeSymlink != 0 { + return fmt.Errorf("destination exists and is not a symlink") + } + } + + // The source path must exist and be a directory to be usable. + if fi, err := os.Stat(u.Path); err != nil { + return fmt.Errorf("source path error: %s", err) + } else if !fi.IsDir() { + return fmt.Errorf("source path must be a directory") + } + + // Create all the parent directories + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + return os.Symlink(u.Path, dst) +} diff --git a/config/module/module.go b/config/module/module.go new file mode 100644 index 0000000000..f8649f6e9d --- /dev/null +++ b/config/module/module.go @@ -0,0 +1,7 @@ +package module + +// Module represents the metadata for a single module. +type Module struct { + Name string + Source string +} diff --git a/config/module/module_test.go b/config/module/module_test.go new file mode 100644 index 0000000000..01035f2696 --- /dev/null +++ b/config/module/module_test.go @@ -0,0 +1,21 @@ +package module + +import ( + "net/url" + "path/filepath" +) + +const fixtureDir = "./test-fixtures" + +func testModule(n string) string { + p := filepath.Join(fixtureDir, n) + p, err := filepath.Abs(p) + if err != nil { + panic(err) + } + + var url url.URL + url.Scheme = "file" + url.Path = p + return url.String() +} diff --git a/config/module/storage.go b/config/module/storage.go new file mode 100644 index 0000000000..14b5181e54 --- /dev/null +++ b/config/module/storage.go @@ -0,0 +1,13 @@ +package module + +// Storage is an interface that knows how to lookup downloaded modules +// as well as download and update modules from their sources into the +// proper location. +type Storage interface { + // Dir returns the directory on local disk where the modulue source + // can be loaded from. + Dir(string) (string, bool, error) + + // Get will download and optionally update the given module. + Get(string, bool) error +} diff --git a/config/module/test-fixtures/basic/main.tf b/config/module/test-fixtures/basic/main.tf new file mode 100644 index 0000000000..fec56017dc --- /dev/null +++ b/config/module/test-fixtures/basic/main.tf @@ -0,0 +1 @@ +# Hello diff --git a/config/module/tree.go b/config/module/tree.go new file mode 100644 index 0000000000..851057c0f0 --- /dev/null +++ b/config/module/tree.go @@ -0,0 +1,71 @@ +package module + +import ( + "github.com/hashicorp/terraform/config" +) + +// Tree represents the module import tree of configurations. +// +// This Tree structure can be used to get (download) new modules, load +// all the modules without getting, flatten the tree into something +// Terraform can use, etc. +type Tree struct { + Config *config.Config + Children []*Tree +} + +// GetMode is an enum that describes how modules are loaded. +// +// GetModeLoad says that modules will not be downloaded or updated, they will +// only be loaded from the storage. +// +// GetModeGet says that modules can be initially downloaded if they don't +// exist, but otherwise to just load from the current version in storage. +// +// GetModeUpdate says that modules should be checked for updates and +// downloaded prior to loading. If there are no updates, we load the version +// from disk, otherwise we download first and then load. +type GetMode byte + +const ( + GetModeNone GetMode = iota + GetModeGet + GetModeUpdate +) + +// Flatten takes the entire module tree and flattens it into a single +// namespace in *config.Config with no module imports. +// +// Validate is called here implicitly, since it is important that semantic +// checks pass before flattening the configuration. Otherwise, encapsulation +// breaks in horrible ways and the errors that come out the other side +// will be surprising. +func (t *Tree) Flatten() (*config.Config, error) { + return nil, nil +} + +// Modules returns the list of modules that this tree imports. +func (t *Tree) Modules() []*Module { + return nil +} + +// Load loads the configuration of the entire tree. +// +// The parameters are used to tell the tree where to find modules and +// whether it can download/update modules along the way. +// +// Various semantic-like checks are made along the way of loading since +// module trees inherently require the configuration to be in a reasonably +// sane state: no circular dependencies, proper module sources, etc. A full +// suite of validations can be done by running Validate (after loading). +func (t *Tree) Load(s Storage, mode GetMode) error { + return nil +} + +// Validate does semantic checks on the entire tree of configurations. +// +// This will call the respective config.Config.Validate() functions as well +// as verifying things such as parameters/outputs between the various modules. +func (t *Tree) Validate() error { + return nil +}