From f723270e3ea1bd96ce2adf664dc73e14dbb5a1d2 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 15 Jun 2017 10:12:00 -0400 Subject: [PATCH 1/3] search the vendor directory for plugins The default location for users to manually add plugins will be ./terraform.d/plugins/ --- command/command.go | 5 ++++ command/init_test.go | 57 ++++++++++++++++++++++++++++++++++++++++ command/plugins.go | 4 +++ plugin/discovery/find.go | 1 + 4 files changed, 67 insertions(+) diff --git a/command/command.go b/command/command.go index 35a85d53bd..2205c53e3d 100644 --- a/command/command.go +++ b/command/command.go @@ -15,6 +15,11 @@ var test bool = false // DefaultDataDir is the default directory for storing local data. const DefaultDataDir = ".terraform" +// DefaultPluginVendorDir is the location in the config directory to look for +// user-added plugin binaries. Terraform only reads from this path if it +// exists, it is never created by terraform. +const DefaultPluginVendorDir = "terraform.d/plugins" + // DefaultStateFilename is the default filename used for the state file. const DefaultStateFilename = "terraform.tfstate" diff --git a/command/init_test.go b/command/init_test.go index 6ccdbadbcf..e17ff88358 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -494,6 +494,63 @@ func TestInit_getProvider(t *testing.T) { } } +// make sure we can locate providers in various paths +func TestInit_findVendoredProviders(t *testing.T) { + // Create a temporary working directory that is empty + td := tempDir(t) + + configDirName := "init-get-providers" + copy.CopyDir(testFixturePath(configDirName), filepath.Join(td, configDirName)) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + c := &InitCommand{ + Meta: m, + providerInstaller: &mockProviderInstaller{}, + } + + // make our plugin paths + if err := os.MkdirAll(c.pluginDir(), 0755); err != nil { + t.Fatal(err) + } + vendorMachineDir := filepath.Join( + DefaultPluginVendorDir, + fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), + ) + if err := os.MkdirAll(vendorMachineDir, 0755); err != nil { + t.Fatal(err) + } + + // add some dummy providers + // the auto plugin directory + exactPath := filepath.Join(c.pluginDir(), "terraform-provider-exact_v1.2.3_x4") + if err := ioutil.WriteFile(exactPath, []byte("test bin"), 0755); err != nil { + t.Fatal(err) + } + // the vendor path + greaterThanPath := filepath.Join(vendorMachineDir, "terraform-provider-greater_than_v2.3.4_x4") + if err := ioutil.WriteFile(greaterThanPath, []byte("test bin"), 0755); err != nil { + t.Fatal(err) + } + // TODO: change this to the -plugin-dir path + betweenPath := filepath.Join(".", "terraform-provider-between_v2.3.4_x4") + if err := ioutil.WriteFile(betweenPath, []byte("test bin"), 0755); err != nil { + t.Fatal(err) + } + + args := []string{configDirName} + if code := c.Run(args); code != 0 { + t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) + } + +} + func TestInit_getUpgradePlugins(t *testing.T) { // Create a temporary working directory that is empty td := tempDir(t) diff --git a/command/plugins.go b/command/plugins.go index 78d07c92f0..75106a9a85 100644 --- a/command/plugins.go +++ b/command/plugins.go @@ -97,10 +97,14 @@ func (m *Meta) pluginDirs(includeAutoInstalled bool) []string { dirs = append(dirs, filepath.Dir(exePath)) } + // add the user vendor directory + dirs = append(dirs, DefaultPluginVendorDir) + if includeAutoInstalled { dirs = append(dirs, m.pluginDir()) } dirs = append(dirs, m.GlobalPluginDirs...) + return dirs } diff --git a/plugin/discovery/find.go b/plugin/discovery/find.go index c28a601835..539b9358c5 100644 --- a/plugin/discovery/find.go +++ b/plugin/discovery/find.go @@ -96,6 +96,7 @@ func findPluginPaths(kind string, machineName string, dirs []string) []string { continue } + // FIXME: we pass in GOOS_GOARCH paths directly, so these may not be "legacy" if strings.HasPrefix(fullName, prefix) { // Legacy style with files directly in the base directory absPath, err := filepath.Abs(filepath.Join(baseDir, fullName)) From 000e8607060ea12e48bff44693d539ce5d8a975b Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 15 Jun 2017 14:26:12 -0400 Subject: [PATCH 2/3] Add plugin dir scaffolding add pluginDir to command.Meta, the flag to initialize it, and the methods to save and restore it. --- command/apply.go | 6 ++++++ command/command.go | 4 ++++ command/init.go | 9 +++++++++ command/meta.go | 6 ++++++ command/plan.go | 6 ++++++ command/plugins.go | 41 +++++++++++++++++++++++++++++++++++++++++ command/refresh.go | 6 ++++++ 7 files changed, 78 insertions(+) diff --git a/command/apply.go b/command/apply.go index 9c8c4904a2..a4bbba7a6e 100644 --- a/command/apply.go +++ b/command/apply.go @@ -65,6 +65,12 @@ func (c *ApplyCommand) Run(args []string) int { return 1 } + // Check for user-supplied plugin path + if c.pluginPath, err = c.loadPluginPath(); err != nil { + c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err)) + return 1 + } + if !c.Destroy && maybeInit { // We need the pwd for the getter operation below pwd, err := os.Getwd() diff --git a/command/command.go b/command/command.go index 2205c53e3d..bb8c7be5c6 100644 --- a/command/command.go +++ b/command/command.go @@ -15,6 +15,10 @@ var test bool = false // DefaultDataDir is the default directory for storing local data. const DefaultDataDir = ".terraform" +// PluginPathFile is the name of the file in the data dir which stores the list +// of directories supplied by the user with the `-plugin-dir` flag during init. +const PluginPathFile = "plugin_path" + // DefaultPluginVendorDir is the location in the config directory to look for // user-added plugin binaries. Terraform only reads from this path if it // exists, it is never created by terraform. diff --git a/command/init.go b/command/init.go index 5d33063e9e..c98eb1f90c 100644 --- a/command/init.go +++ b/command/init.go @@ -32,6 +32,7 @@ type InitCommand struct { func (c *InitCommand) Run(args []string) int { var flagBackend, flagGet, flagGetPlugins, flagUpgrade bool var flagConfigExtra map[string]interface{} + var flagPluginPath FlagStringSlice args = c.Meta.process(args, false) cmdFlags := c.flagSet("init") @@ -44,12 +45,15 @@ func (c *InitCommand) Run(args []string) int { cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout") cmdFlags.BoolVar(&c.reconfigure, "reconfigure", false, "reconfigure") cmdFlags.BoolVar(&flagUpgrade, "upgrade", false, "") + cmdFlags.Var(&flagPluginPath, "plugin-dir", "plugin directory") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 } + c.pluginPath = flagPluginPath + // set getProvider if we don't have a test version already if c.providerInstaller == nil { c.providerInstaller = &discovery.ProviderInstaller{ @@ -67,6 +71,11 @@ func (c *InitCommand) Run(args []string) int { return 1 } + if err := c.storePluginPath(c.pluginPath); err != nil { + c.Ui.Error(fmt.Sprintf("Error saving -plugin-path values: %s", err)) + return 1 + } + // Get our pwd. We don't always need it but always getting it is easier // than the logic to determine if it is or isn't needed. pwd, err := os.Getwd() diff --git a/command/meta.go b/command/meta.go index ffd9ec62e4..4463d20775 100644 --- a/command/meta.go +++ b/command/meta.go @@ -48,6 +48,12 @@ type Meta struct { // DataDir method. dataDir string + // pluginPath is a user defined set of directories to look for plugins. + // This is set during init with the `-plugin-dir` flag, saved to a file in + // the data directory. + // This overrides all other search paths when discoverying plugins. + pluginPath []string + // Override certain behavior for tests within this package testingOverrides *testingOverrides diff --git a/command/plan.go b/command/plan.go index a38a228728..1edc179e55 100644 --- a/command/plan.go +++ b/command/plan.go @@ -45,6 +45,12 @@ func (c *PlanCommand) Run(args []string) int { return 1 } + // Check for user-supplied plugin path + if c.pluginPath, err = c.loadPluginPath(); err != nil { + c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err)) + return 1 + } + // Check if the path is a plan plan, err := c.Plan(configPath) if err != nil { diff --git a/command/plugins.go b/command/plugins.go index 75106a9a85..07a1552551 100644 --- a/command/plugins.go +++ b/command/plugins.go @@ -1,8 +1,11 @@ package command import ( + "encoding/json" "fmt" + "io/ioutil" "log" + "os" "os/exec" "path/filepath" "runtime" @@ -69,6 +72,40 @@ func (r *multiVersionProviderResolver) ResolveProviders( return factories, errs } +// store the user-supplied path for plugin discovery +func (m *Meta) storePluginPath(pluginPath []string) error { + if len(pluginPath) == 0 { + return nil + } + + js, err := json.MarshalIndent(pluginPath, "", " ") + if err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(m.DataDir(), PluginPathFile), js, 0644) +} + +// Load the user-defined plugin search path into Meta.pluginPath if the file +// exists. +func (m *Meta) loadPluginPath() ([]string, error) { + js, err := ioutil.ReadFile(filepath.Join(m.DataDir(), PluginPathFile)) + if os.IsNotExist(err) { + return nil, nil + } + + if err != nil { + return nil, err + } + + var pluginPath []string + if err := json.Unmarshal(js, &pluginPath); err != nil { + return nil, err + } + + return pluginPath, nil +} + // the default location for automatically installed plugins func (m *Meta) pluginDir() string { return filepath.Join(m.DataDir(), "plugins", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) @@ -80,6 +117,10 @@ func (m *Meta) pluginDir() string { // of the same plugin version are found, but newer versions always override // older versions where both satisfy the provider version constraints. func (m *Meta) pluginDirs(includeAutoInstalled bool) []string { + // user defined paths take precedence + if len(m.pluginPath) > 0 { + return m.pluginPath + } // When searching the following directories, earlier entries get precedence // if the same plugin version is found twice, but newer versions will diff --git a/command/refresh.go b/command/refresh.go index 72623ed45e..7ffa9d8904 100644 --- a/command/refresh.go +++ b/command/refresh.go @@ -44,6 +44,12 @@ func (c *RefreshCommand) Run(args []string) int { return 1 } + // Check for user-supplied plugin path + if c.pluginPath, err = c.loadPluginPath(); err != nil { + c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err)) + return 1 + } + var conf *config.Config if mod != nil { conf = mod.Config() From be2069ac81b0ee8f6e5902185d8decb08be87df9 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Thu, 15 Jun 2017 15:23:16 -0400 Subject: [PATCH 3/3] add -plugin-dir option The -plugin-dir option lets the user specify custom search paths for plugins. This overrides all other plugin search paths, and prevents the auto-installation of plugins. We also make sure that the availability of plugins is always checked during init, even if -get-plugins=false or -plugin-dir is set. --- command/init.go | 86 ++++++++++++++++++++-------------- command/init_test.go | 101 +++++++++++++++++++++++++++++++++++++++- command/plugins.go | 3 ++ command/plugins_test.go | 25 ++++++++++ 4 files changed, 179 insertions(+), 36 deletions(-) diff --git a/command/init.go b/command/init.go index c98eb1f90c..69c0ec3aaf 100644 --- a/command/init.go +++ b/command/init.go @@ -22,6 +22,9 @@ import ( type InitCommand struct { Meta + // getPlugins is for the -get-plugins flag + getPlugins bool + // providerInstaller is used to download and install providers that // aren't found locally. This uses a discovery.ProviderInstaller instance // by default, but it can be overridden here as a way to mock fetching @@ -30,7 +33,7 @@ type InitCommand struct { } func (c *InitCommand) Run(args []string) int { - var flagBackend, flagGet, flagGetPlugins, flagUpgrade bool + var flagBackend, flagGet, flagUpgrade bool var flagConfigExtra map[string]interface{} var flagPluginPath FlagStringSlice @@ -39,7 +42,7 @@ func (c *InitCommand) Run(args []string) int { cmdFlags.BoolVar(&flagBackend, "backend", true, "") cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "") cmdFlags.BoolVar(&flagGet, "get", true, "") - cmdFlags.BoolVar(&flagGetPlugins, "get-plugins", true, "") + cmdFlags.BoolVar(&c.getPlugins, "get-plugins", true, "") cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data") cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state") cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout") @@ -52,7 +55,10 @@ func (c *InitCommand) Run(args []string) int { return 1 } - c.pluginPath = flagPluginPath + if len(flagPluginPath) > 0 { + c.pluginPath = flagPluginPath + c.getPlugins = false + } // set getProvider if we don't have a test version already if c.providerInstaller == nil { @@ -141,7 +147,7 @@ func (c *InitCommand) Run(args []string) int { // If we're requesting backend configuration or looking for required // plugins, load the backend - if flagBackend || flagGetPlugins { + if flagBackend { header = true // Only output that we're initializing a backend if we have @@ -165,31 +171,29 @@ func (c *InitCommand) Run(args []string) int { } } - // Now that we have loaded all modules, check the module tree for missing providers - if flagGetPlugins { - sMgr, err := back.State(c.Workspace()) - if err != nil { - c.Ui.Error(fmt.Sprintf( - "Error loading state: %s", err)) - return 1 - } + // Now that we have loaded all modules, check the module tree for missing providers. + sMgr, err := back.State(c.Workspace()) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error loading state: %s", err)) + return 1 + } - if err := sMgr.RefreshState(); err != nil { - c.Ui.Error(fmt.Sprintf( - "Error refreshing state: %s", err)) - return 1 - } + if err := sMgr.RefreshState(); err != nil { + c.Ui.Error(fmt.Sprintf( + "Error refreshing state: %s", err)) + return 1 + } - c.Ui.Output(c.Colorize().Color( - "[reset][bold]Initializing provider plugins...", - )) + c.Ui.Output(c.Colorize().Color( + "[reset][bold]Initializing provider plugins...", + )) - err = c.getProviders(path, sMgr.State(), flagUpgrade) - if err != nil { - // this function provides its own output - log.Printf("[ERROR] %s", err) - return 1 - } + err = c.getProviders(path, sMgr.State(), flagUpgrade) + if err != nil { + // this function provides its own output + log.Printf("[ERROR] %s", err) + return 1 } // If we outputted information, then we need to output a newline @@ -229,17 +233,26 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade missing := c.missingPlugins(available, requirements) var errs error - for provider, reqd := range missing { - c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider)) - _, err := c.providerInstaller.Get(provider, reqd.Versions) + if c.getPlugins { + for provider, reqd := range missing { + c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider)) + _, err := c.providerInstaller.Get(provider, reqd.Versions) + + if err != nil { + c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions)) + errs = multierror.Append(errs, err) + } + } - if err != nil { + if errs != nil { + return errs + } + } else if len(missing) > 0 { + // we have missing providers, but aren't going to try and download them + for provider, reqd := range missing { c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions)) - errs = multierror.Append(errs, err) + errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider)) } - } - - if errs != nil { return errs } @@ -360,6 +373,11 @@ Options: -no-color If specified, output won't contain any color. + -plugin-dir Directory containing plugin binaries. This overrides all + default search paths for plugins, and prevents the + automatic installation of plugins. This flag can be used + multiple times. + -reconfigure Reconfigure the backend, ignoring any saved configuration. -upgrade=false If installing modules (-get) or plugins (-get-plugins), diff --git a/command/init_test.go b/command/init_test.go index e17ff88358..dd296d9458 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -538,7 +538,7 @@ func TestInit_findVendoredProviders(t *testing.T) { if err := ioutil.WriteFile(greaterThanPath, []byte("test bin"), 0755); err != nil { t.Fatal(err) } - // TODO: change this to the -plugin-dir path + // Check the current directory too betweenPath := filepath.Join(".", "terraform-provider-between_v2.3.4_x4") if err := ioutil.WriteFile(betweenPath, []byte("test bin"), 0755); err != nil { t.Fatal(err) @@ -548,7 +548,6 @@ func TestInit_findVendoredProviders(t *testing.T) { if code := c.Run(args); code != 0 { t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) } - } func TestInit_getUpgradePlugins(t *testing.T) { @@ -767,3 +766,101 @@ func TestInit_providerLockFile(t *testing.T) { t.Errorf("wrong provider lock file contents\ngot: %s\nwant: %s", buf, wantLockFile) } } + +// Test user-supplied -plugin-dir +func TestInit_pluginDirProviders(t *testing.T) { + td := tempDir(t) + copy.CopyDir(testFixturePath("init-get-providers"), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + c := &InitCommand{ + Meta: m, + providerInstaller: &mockProviderInstaller{}, + } + + // make our vendor paths + pluginPath := []string{"a", "b", "c"} + for _, p := range pluginPath { + if err := os.MkdirAll(p, 0755); err != nil { + t.Fatal(err) + } + } + + // add some dummy providers in our plugin dirs + for i, name := range []string{ + "terraform-provider-exact_v1.2.3_x4", + "terraform-provider-greater_than_v2.3.4_x4", + "terraform-provider-between_v2.3.4_x4", + } { + + if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil { + t.Fatal(err) + } + } + + args := []string{ + "-plugin-dir", "a", + "-plugin-dir", "b", + "-plugin-dir", "c", + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: \n%s", ui.ErrorWriter) + } +} + +// Test user-supplied -plugin-dir doesn't allow auto-install +func TestInit_pluginDirProvidersDoesNotGet(t *testing.T) { + td := tempDir(t) + copy.CopyDir(testFixturePath("init-get-providers"), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + c := &InitCommand{ + Meta: m, + providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) { + t.Fatalf("plugin installer should not have been called for %q", provider) + return discovery.PluginMeta{}, nil + }), + } + + // make our vendor paths + pluginPath := []string{"a", "b"} + for _, p := range pluginPath { + if err := os.MkdirAll(p, 0755); err != nil { + t.Fatal(err) + } + } + + // add some dummy providers in our plugin dirs + for i, name := range []string{ + "terraform-provider-exact_v1.2.3_x4", + "terraform-provider-greater_than_v2.3.4_x4", + } { + + if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil { + t.Fatal(err) + } + } + + args := []string{ + "-plugin-dir", "a", + "-plugin-dir", "b", + } + if code := c.Run(args); code == 0 { + // should have been an error + t.Fatalf("bad: \n%s", ui.OutputWriter) + } +} diff --git a/command/plugins.go b/command/plugins.go index 07a1552551..6ce6427274 100644 --- a/command/plugins.go +++ b/command/plugins.go @@ -83,6 +83,9 @@ func (m *Meta) storePluginPath(pluginPath []string) error { return err } + // if this fails, so will WriteFile + os.MkdirAll(m.DataDir(), 0755) + return ioutil.WriteFile(filepath.Join(m.DataDir(), PluginPathFile), js, 0644) } diff --git a/command/plugins_test.go b/command/plugins_test.go index 2ee466d284..e0cd13c5bd 100644 --- a/command/plugins_test.go +++ b/command/plugins_test.go @@ -2,12 +2,37 @@ package command import ( "fmt" + "io/ioutil" "os" "path/filepath" + "reflect" + "testing" "github.com/hashicorp/terraform/plugin/discovery" ) +func TestPluginPath(t *testing.T) { + td, err := ioutil.TempDir("", "tf") + defer os.RemoveAll(td) + defer testChdir(t, td)() + + pluginPath := []string{"a", "b", "c"} + + m := Meta{} + if err := m.storePluginPath(pluginPath); err != nil { + t.Fatal(err) + } + + restoredPath, err := m.loadPluginPath() + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(pluginPath, restoredPath) { + t.Fatalf("expected plugin path %#v, got %#v", pluginPath, restoredPath) + } +} + // mockProviderInstaller is a discovery.PluginInstaller implementation that // is a mock for discovery.ProviderInstaller. type mockProviderInstaller struct {