diff --git a/internal/getproviders/types.go b/internal/getproviders/types.go index 0af6ebd831..9d720be665 100644 --- a/internal/getproviders/types.go +++ b/internal/getproviders/types.go @@ -8,6 +8,8 @@ import ( "strings" "github.com/apparentlymart/go-versions/versions" + "github.com/apparentlymart/go-versions/versions/constraints" + "github.com/hashicorp/terraform/addrs" ) @@ -23,6 +25,10 @@ type VersionList = versions.List // by the end-user. type VersionSet = versions.Set +// VersionConstraints represents a set of version constraints, which can +// define the membership of a VersionSet by exclusion. +type VersionConstraints = constraints.IntersectionSpec + // ParseVersion parses a "semver"-style version string into a Version value, // which is the version syntax we use for provider versions. func ParseVersion(str string) (Version, error) { @@ -246,3 +252,56 @@ func (l PackageMetaList) FilterProviderPlatformExactVersion(provider addrs.Provi } return ret } + +// VersionConstraintsString returns a UI-oriented string representation of +// a VersionConstraints value. +func VersionConstraintsString(spec VersionConstraints) string { + // (we have our own function for this because the upstream versions + // library prefers to use npm/cargo-style constraint syntax, but + // Terraform prefers Ruby-like. Maybe we can upstream a "RubyLikeString") + // function to do this later, but having this in here avoids blocking on + // that and this is the sort of thing that is unlikely to need ongoing + // maintenance because the version constraint syntax is unlikely to change.) + + var b strings.Builder + for i, sel := range spec { + if i > 0 { + b.WriteString(", ") + } + switch sel.Operator { + case constraints.OpGreaterThan: + b.WriteString("> ") + case constraints.OpLessThan: + b.WriteString("< ") + case constraints.OpGreaterThanOrEqual: + b.WriteString(">= ") + case constraints.OpGreaterThanOrEqualPatchOnly, constraints.OpGreaterThanOrEqualMinorOnly: + // These two differ in how the version is written, not in the symbol. + b.WriteString("~> ") + case constraints.OpLessThanOrEqual: + b.WriteString("<= ") + case constraints.OpEqual: + b.WriteString("") + case constraints.OpNotEqual: + b.WriteString("!= ") + default: + // The above covers all of the operators we support during + // parsing, so we should not get here. + b.WriteString("??? ") + } + + if sel.Operator == constraints.OpGreaterThanOrEqualMinorOnly { + // The minor-pessimistic syntax uses only two version components. + fmt.Fprintf(&b, "%s.%s", sel.Boundary.Major, sel.Boundary.Minor) + } else { + fmt.Fprintf(&b, "%s.%s.%s", sel.Boundary.Major, sel.Boundary.Minor, sel.Boundary.Patch) + } + if sel.Boundary.Prerelease != "" { + b.WriteString("-" + sel.Boundary.Prerelease) + } + if sel.Boundary.Metadata != "" { + b.WriteString("+" + sel.Boundary.Metadata) + } + } + return b.String() +} diff --git a/internal/providercache/installer.go b/internal/providercache/installer.go index 903d9b6147..72ef591063 100644 --- a/internal/providercache/installer.go +++ b/internal/providercache/installer.go @@ -1,7 +1,351 @@ package providercache +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/apparentlymart/go-versions/versions" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/internal/copydir" + "github.com/hashicorp/terraform/internal/getproviders" +) + // Installer is the main type in this package, representing a provider installer // with a particular configuration-specific cache directory and an optional // global cache directory. type Installer struct { + // targetDir is the cache directory we're ultimately aiming to get the + // requested providers installed into. + targetDir *Dir + + // source is the provider source that the installer will use to discover + // what provider versions are available for installation and to + // find the source locations for any versions that are not already + // available via one of the cache directories. + source getproviders.Source + + // globalCacheDir is an optional additional directory that will, if + // provided, be treated as a read-through cache when retrieving new + // provider versions. That is, new packages are fetched into this + // directory first and then linked into targetDir, which allows sharing + // both the disk space and the download time for a particular provider + // version between different configurations on the same system. + globalCacheDir *Dir +} + +// NewInstaller constructs and returns a new installer with the given target +// directory and provider source. +// +// A newly-created installer does not have a global cache directory configured, +// but a caller can make a follow-up call to SetGlobalCacheDir to provide +// one prior to taking any installation actions. +// +// The target directory MUST NOT also be an input consulted by the given source, +// or the result is undefined. +func NewInstaller(targetDir *Dir, source getproviders.Source) *Installer { + return &Installer{ + targetDir: targetDir, + source: source, + } +} + +// SetGlobalCacheDir activates a second tier of caching for the receiving +// installer, with the given directory used as a read-through cache for +// installation operations that need to retrieve new packages. +// +// The global cache directory for an installer must never be the same as its +// target directory, and must not be used as one of its provider sources. +// If these overlap then undefined behavior will result. +func (i *Installer) SetGlobalCacheDir(cacheDir *Dir) { + // A little safety check to catch straightforward mistakes where the + // directories overlap. Better to panic early than to do + // possibly-distructive actions on the cache directory downstream. + if same, err := copydir.SameFile(i.targetDir.baseDir, cacheDir.baseDir); err == nil && !same { + panic(fmt.Sprintf("global cache directory %s must not match the installation target directory", i.targetDir.baseDir)) + } + i.globalCacheDir = cacheDir +} + +// EnsureProviderVersions compares the given provider requirements with what +// is already available in the installer's target directory and then takes +// appropriate installation actions to ensure that suitable packages +// are available in the target cache directory. +// +// The given mode modifies how the operation will treat providers that already +// have acceptable versions available in the target cache directory. See the +// documentation for InstallMode and the InstallMode values for more +// information. +// +// The given context can be used to cancel the overall installation operation +// (causing any operations in progress to fail with an error), and can also +// include an InstallerEvents value for optional intermediate progress +// notifications. +// +// If a given InstallerEvents subscribes to notifications about installation +// failures then those notifications will be redundant with the ones included +// in the final returned error value so callers should show either one or the +// other, and not both. +func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs map[addrs.Provider]getproviders.VersionConstraints, mode InstallMode) (map[addrs.Provider]getproviders.Version, error) { + // FIXME: Currently the context isn't actually propagated into the + // other functions we call here, because they are not context-aware. + // Right now the context is used only for the InstallerEvents object. + // Before considering this "finished" we should update the functions + // we're calling below that might perform external network requests + // and make them also take a context and respect cancellation of it. + + errs := map[addrs.Provider]error{} + evts := installerEventsForContext(ctx) + + if cb := evts.PendingProviders; cb != nil { + cb(reqs) + } + + // Here we'll keep track of which exact version we've selected for each + // provider in the requirements. + selected := map[addrs.Provider]getproviders.Version{} + + // Step 1: Which providers might we need to fetch a new version of? + // This produces the subset of requirements we need to ask the provider + // source about. + have := i.targetDir.AllAvailablePackages() + mightNeed := map[addrs.Provider]getproviders.VersionSet{} +MightNeedProvider: + for provider, versionConstraints := range reqs { + acceptableVersions := versions.MeetingConstraints(versionConstraints) + if mode.forceQueryAllProviders() { + // If our mode calls for us to look for newer versions regardless + // of whether an existing version is acceptable, we "might need" + // _all_ of the requested providers. + mightNeed[provider] = acceptableVersions + continue + } + havePackages, ok := have[provider] + if !ok { // If we don't have any versions at all then we'll definitely need it + mightNeed[provider] = acceptableVersions + continue + } + // If we already have some versions installed and our mode didn't + // force us to check for new ones anyway then we'll check only if + // there isn't already at least one version in our cache that is + // in the set of acceptable versions. + for _, pkg := range havePackages { + if acceptableVersions.Has(pkg.Version) { + // We will take no further actions for this provider, because + // a version we have is already acceptable. + selected[provider] = pkg.Version + if cb := evts.ProviderAlreadyInstalled; cb != nil { + cb(provider, pkg.Version) + } + continue MightNeedProvider + } + } + // If we get here then we didn't find any cached version that is + // in our set of acceptable versions. + mightNeed[provider] = acceptableVersions + } + + // Step 2: Query the provider source for each of the providers we selected + // in the first step and select the latest available version that is + // in the set of acceptable versions. + // + // This produces a set of packages to install to our cache in the next step. + need := map[addrs.Provider]getproviders.Version{} +NeedProvider: + for provider, acceptableVersions := range mightNeed { + if cb := evts.QueryPackagesBegin; cb != nil { + cb(provider, reqs[provider]) + } + available, err := i.source.AvailableVersions(provider) + if err != nil { + // TODO: Consider retrying a few times for certain types of + // source errors that seem likely to be transient. + errs[provider] = err + if cb := evts.QueryPackagesFailure; cb != nil { + cb(provider, err) + } + // We will take no further actions for this provider. + continue + } + available.Sort() // put the versions in increasing order of precedence + for i := len(available) - 1; i >= 0; i-- { // walk backwards to consider newer versions first + if acceptableVersions.Has(available[i]) { + need[provider] = available[i] + if cb := evts.QueryPackagesSuccess; cb != nil { + cb(provider, available[i]) + } + continue NeedProvider + } + } + // If we get here then the source has no packages that meet the given + // version constraint, which we model as a query error. + err = fmt.Errorf("no available releases match the given constraints %s", getproviders.VersionConstraintsString(reqs[provider])) + errs[provider] = err + if cb := evts.QueryPackagesFailure; cb != nil { + cb(provider, err) + } + } + + // Step 3: For each provider version we've decided we need to install, + // install its package into our target cache (possibly via the global cache). + targetPlatform := i.targetDir.targetPlatform // we inherit this to behave correctly in unit tests + for provider, version := range need { + if i.globalCacheDir != nil { + // Step 3a: If our global cache already has this version available then + // we'll just link it in. + if cached := i.globalCacheDir.ProviderVersion(provider, version); cached != nil { + if cb := evts.LinkFromCacheBegin; cb != nil { + cb(provider, version, i.globalCacheDir.baseDir) + } + err := i.targetDir.LinkFromOtherCache(cached) + if err != nil { + errs[provider] = err + if cb := evts.LinkFromCacheFailure; cb != nil { + cb(provider, version, err) + } + continue + } + // We'll fetch what we just linked to make sure it actually + // did show up there. + new := i.targetDir.ProviderVersion(provider, version) + if new == nil { + err := fmt.Errorf("after linking %s from provider cache at %s it is still not detected in the target directory; this is a bug in Terraform", provider, i.globalCacheDir.baseDir) + if cb := evts.LinkFromCacheFailure; cb != nil { + cb(provider, version, err) + } + continue + } + selected[provider] = version + if cb := evts.LinkFromCacheSuccess; cb != nil { + cb(provider, version, new.PackageDir) + } + continue // Don't need to do full install, then. + } + } + + // Step 3b: Get the package metadata for the selected version from our + // provider source. + // + // This is the step where we might detect and report that the provider + // isn't available for the current platform. + if cb := evts.FetchPackageMeta; cb != nil { + cb(provider, version) + } + meta, err := i.source.PackageMeta(provider, version, targetPlatform) + if err != nil { + errs[provider] = err + if cb := evts.FetchPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + + // Step 3c: Retrieve the package indicated by the metadata we received, + // either directly into our target directory or via the global cache + // directory. + if cb := evts.FetchPackageBegin; cb != nil { + cb(provider, version, meta.Location) + } + var installTo, linkTo *Dir + if i.globalCacheDir != nil { + installTo = i.globalCacheDir + linkTo = i.targetDir + } else { + installTo = i.targetDir + linkTo = nil // no linking needed + } + err = installTo.InstallPackage(meta) + if err != nil { + // TODO: Consider retrying for certain kinds of error that seem + // likely to be transient. For now, we just treat all errors equally. + errs[provider] = err + if cb := evts.FetchPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + new := installTo.ProviderVersion(provider, version) + if new == nil { + err := fmt.Errorf("after installing %s it is still not detected in the target directory; this is a bug in Terraform", provider) + if cb := evts.FetchPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + if linkTo != nil { + // We skip emitting the "LinkFromCache..." events here because + // it's simpler for the caller to treat them as mutually exclusive. + // We can just subsume the linking step under the "FetchPackage..." + // series here (and that's why we use FetchPackageFailure below). + err := linkTo.LinkFromOtherCache(new) + if err != nil { + errs[provider] = err + if cb := evts.FetchPackageFailure; cb != nil { + cb(provider, version, err) + } + continue + } + } + selected[provider] = version + if cb := evts.FetchPackageSuccess; cb != nil { + cb(provider, version, new.PackageDir) + } + } + + if len(errs) > 0 { + return selected, InstallerError{ + ProviderErrors: errs, + } + } + return selected, nil +} + +// InstallMode customizes the details of how an install operation treats +// providers that have versions already cached in the target directory. +type InstallMode rune + +const ( + // InstallNewProvidersOnly is an InstallMode that causes the installer + // to accept any existing version of a requested provider that is already + // cached as long as it's in the given version sets, without checking + // whether new versions are available that are also in the given version + // sets. + InstallNewProvidersOnly InstallMode = 'N' + + // InstallUpgrades is an InstallMode that causes the installer to check + // all requested providers to see if new versions are available that + // are also in the given version sets, even if a suitable version of + // a given provider is already available. + InstallUpgrades InstallMode = 'U' +) + +func (m InstallMode) forceQueryAllProviders() bool { + return m == InstallUpgrades +} + +// InstallerError is an error type that may be returned (but is not guaranteed) +// from Installer.EnsureProviderVersions to indicate potentially several +// separate failed installation outcomes for different providers included in +// the overall request. +type InstallerError struct { + ProviderErrors map[addrs.Provider]error +} + +func (err InstallerError) Error() string { + addrs := make([]addrs.Provider, 0, len(err.ProviderErrors)) + for addr := range err.ProviderErrors { + addrs = append(addrs, addr) + } + sort.Slice(addrs, func(i, j int) bool { + return addrs[i].LessThan(addrs[j]) + }) + var b strings.Builder + b.WriteString("some providers could not be installed:\n") + for _, addr := range addrs { + providerErr := err.ProviderErrors[addr] + fmt.Fprintf(&b, "- %s: %s\n", addr, providerErr.Error()) + } + return b.String() } diff --git a/internal/providercache/installer_events.go b/internal/providercache/installer_events.go index c02d5bd93a..593cbe1665 100644 --- a/internal/providercache/installer_events.go +++ b/internal/providercache/installer_events.go @@ -29,11 +29,15 @@ type InstallerEvents struct { // A recipient driving a UI might, for example, use this to pre-allocate // UI space for status reports for all of the providers and then update // those positions in-place as other events arrive. - PendingProviders func(provider addrs.Provider) + PendingProviders func(reqs map[addrs.Provider]getproviders.VersionConstraints) // ProviderAlreadyInstalled is called for any provider that was included // in PendingProviders but requires no further action because a suitable // version is already present in the local provider cache directory. + // + // This event can also appear after the QueryPackages... series if + // querying determines that a version already available is the newest + // available version. ProviderAlreadyInstalled func(provider addrs.Provider, selectedVersion getproviders.Version) // The QueryPackages... family of events delimit the operation of querying @@ -44,8 +48,13 @@ type InstallerEvents struct { // A particular install operation includes only one query per distinct // provider, so a caller can use the provider argument as a unique // identifier to correlate between successive events. - QueryPackagesBegin func(provider addrs.Provider, versionSet getproviders.VersionSet) + // + // The Begin, Success, and Failure events will each occur only once per + // distinct provider. The Retry event can occur zero or more times, and + // signals a failure that the installer is considering transient. + QueryPackagesBegin func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints) QueryPackagesSuccess func(provider addrs.Provider, selectedVersion getproviders.Version) + QueryPackagesRetry func(provider addrs.Provider, err error) QueryPackagesFailure func(provider addrs.Provider, err error) // The LinkFromCache... family of events delimit the operation of linking @@ -75,8 +84,14 @@ type InstallerEvents struct { // // A particular provider will either notify the LinkFromCache... events // or the FetchPackage... events, never both in the same install operation. + // + // The Query, Begin, Success, and Failure events will each occur only once + // per distinct provider. The Retry event can occur zero or more times, and + // signals a failure that the installer is considering transient. + FetchPackageMeta func(provider addrs.Provider, version getproviders.Version) // fetching metadata prior to real download FetchPackageBegin func(provider addrs.Provider, version getproviders.Version, location getproviders.PackageLocation) FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string) + FetchPackageRetry func(provider addrs.Provider, version getproviders.Version, err error) FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error) }