Merge pull request #6595 from rickard-von-essen/os-clouds-public

Update gophercloud/utils to add support for clouds-public.yaml

vendor/github.com/gophercloud/utils/openstack/clientconfig/requests.go

@@ -3,6 +3,7 @@ package clientconfig
 import (
 	"fmt"
 	"os"
+	"reflect"
 	"strings"
 
 	"github.com/gophercloud/gophercloud"
@@ -15,14 +16,20 @@ import (
 type AuthType string
 
 const (
+	// AuthPassword defines an unknown version of the password
 	AuthPassword AuthType = "password"
-	AuthToken    AuthType = "token"
+	// AuthToken defined an unknown version of the token
+	AuthToken AuthType = "token"
 
+	// AuthV2Password defines version 2 of the password
 	AuthV2Password AuthType = "v2password"
-	AuthV2Token    AuthType = "v2token"
+	// AuthV2Token defines version 2 of the token
+	AuthV2Token AuthType = "v2token"
 
+	// AuthV3Password defines version 3 of the password
 	AuthV3Password AuthType = "v3password"
-	AuthV3Token    AuthType = "v3token"
+	// AuthV3Token defines version 3 of the token
+	AuthV3Token AuthType = "v3token"
 )
 
 // ClientOpts represents options to customize the way a client is
@@ -41,11 +48,16 @@ type ClientOpts struct {
 	// AuthInfo defines the authentication information needed to
 	// authenticate to a cloud when clouds.yaml isn't used.
 	AuthInfo *AuthInfo
+
+	// RegionName is the region to create a Service Client in.
+	// This will override a region in clouds.yaml or can be used
+	// when authenticating directly with AuthInfo.
+	RegionName string
 }
 
-// LoadYAML will load a clouds.yaml file and return the full config.
-func LoadYAML() (map[string]Cloud, error) {
-	content, err := findAndReadYAML()
+// LoadCloudsYAML will load a clouds.yaml file and return the full config.
+func LoadCloudsYAML() (map[string]Cloud, error) {
+	content, err := findAndReadCloudsYAML()
 	if err != nil {
 		return nil, err
 	}
@@ -59,9 +71,52 @@ func LoadYAML() (map[string]Cloud, error) {
 	return clouds.Clouds, nil
 }
 
+// LoadSecureCloudsYAML will load a secure.yaml file and return the full config.
+func LoadSecureCloudsYAML() (map[string]Cloud, error) {
+	var secureClouds Clouds
+
+	content, err := findAndReadSecureCloudsYAML()
+	if err != nil {
+		if err.Error() == "no secure.yaml file found" {
+			// secure.yaml is optional so just ignore read error
+			return secureClouds.Clouds, nil
+		}
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(content, &secureClouds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal yaml: %v", err)
+	}
+
+	return secureClouds.Clouds, nil
+}
+
+// LoadPublicCloudsYAML will load a public-clouds.yaml file and return the full config.
+func LoadPublicCloudsYAML() (map[string]Cloud, error) {
+	var publicClouds PublicClouds
+
+	content, err := findAndReadPublicCloudsYAML()
+	if err != nil {
+		if err.Error() == "no clouds-public.yaml file found" {
+			// clouds-public.yaml is optional so just ignore read error
+			return publicClouds.Clouds, nil
+		}
+
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(content, &publicClouds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal yaml: %v", err)
+	}
+
+	return publicClouds.Clouds, nil
+}
+
 // GetCloudFromYAML will return a cloud entry from a clouds.yaml file.
 func GetCloudFromYAML(opts *ClientOpts) (*Cloud, error) {
-	clouds, err := LoadYAML()
+	clouds, err := LoadCloudsYAML()
 	if err != nil {
 		return nil, fmt.Errorf("unable to load clouds.yaml: %s", err)
 	}
@@ -101,10 +156,74 @@ func GetCloudFromYAML(opts *ClientOpts) (*Cloud, error) {
 		}
 	}
 
+	var cloudIsInCloudsYaml bool
 	if cloud == nil {
-		return nil, fmt.Errorf("Unable to determine a valid entry in clouds.yaml")
+		// not an immediate error as it might still be defined in secure.yaml
+		cloudIsInCloudsYaml = false
+	} else {
+		cloudIsInCloudsYaml = true
 	}
 
+	publicClouds, err := LoadPublicCloudsYAML()
+	if err != nil {
+		return nil, fmt.Errorf("unable to load clouds-public.yaml: %s", err)
+	}
+
+	var profileName = defaultIfEmpty(cloud.Profile, cloud.Cloud)
+	if profileName != "" {
+		publicCloud, ok := publicClouds[profileName]
+		if !ok {
+			return nil, fmt.Errorf("cloud %s does not exist in clouds-public.yaml", profileName)
+		}
+		cloud, err = mergeClouds(cloud, publicCloud)
+		if err != nil {
+			return nil, fmt.Errorf("Could not merge information from clouds.yaml and clouds-public.yaml for cloud %s", profileName)
+		}
+	}
+
+	secureClouds, err := LoadSecureCloudsYAML()
+	if err != nil {
+		return nil, fmt.Errorf("unable to load secure.yaml: %s", err)
+	}
+
+	if secureClouds != nil {
+		// If no entry was found in clouds.yaml, no cloud name was specified,
+		// and only one secureCloud entry exists, use that as the cloud entry.
+		if !cloudIsInCloudsYaml && cloudName == "" && len(secureClouds) == 1 {
+			for _, v := range secureClouds {
+				cloud = &v
+			}
+		}
+
+		secureCloud, ok := secureClouds[cloudName]
+		if !ok && cloud == nil {
+			// cloud == nil serves two purposes here:
+			// if no entry in clouds.yaml was found and
+			// if a single-entry secureCloud wasn't used.
+			// At this point, no entry could be determined at all.
+			return nil, fmt.Errorf("Could not find cloud %s", cloudName)
+		}
+
+		// If secureCloud has content and it differs from the cloud entry,
+		// merge the two together.
+		if !reflect.DeepEqual((Cloud{}), secureCloud) && !reflect.DeepEqual(cloud, secureCloud) {
+			cloud, err = mergeClouds(secureCloud, cloud)
+			if err != nil {
+				return nil, fmt.Errorf("unable to merge information from clouds.yaml and secure.yaml")
+			}
+		}
+	}
+
+	// Default is to verify SSL API requests
+	if cloud.Verify == nil {
+		iTrue := true
+		cloud.Verify = &iTrue
+	}
+
+	// TODO: this is where reading vendor files should go be considered when not found in
+	// clouds-public.yml
+	// https://github.com/openstack/openstacksdk/tree/master/openstack/config/vendors
+
 	return cloud, nil
 }
 
@@ -472,11 +591,20 @@ func NewServiceClient(service string, opts *ClientOpts) (*gophercloud.ServiceCli
 	}
 
 	// Determine the region to use.
+	// First, see if the cloud entry has one.
 	var region string
 	if v := cloud.RegionName; v != "" {
-		region = cloud.RegionName
+		region = v
+	}
+
+	// Next, see if one was specified in the ClientOpts.
+	// If so, this takes precedence.
+	if v := opts.RegionName; v != "" {
+		region = v
 	}
 
+	// Finally, see if there's an environment variable.
+	// This should always override prior settings.
 	if v := os.Getenv(envPrefix + "REGION_NAME"); v != "" {
 		region = v
 	}

vendor/github.com/gophercloud/utils/openstack/clientconfig/results.go

@@ -1,5 +1,12 @@
 package clientconfig
 
+// PublicClouds represents a collection of PublicCloud entries in clouds-public.yaml file.
+// The format of the clouds-public.yml is documented at
+// https://docs.openstack.org/python-openstackclient/latest/configuration/
+type PublicClouds struct {
+	Clouds map[string]Cloud `yaml:"public-clouds"`
+}
+
 // Clouds represents a collection of Cloud entries in a clouds.yaml file.
 // The format of clouds.yaml is documented at
 // https://docs.openstack.org/os-client-config/latest/user/configuration.html.
@@ -7,8 +14,10 @@ type Clouds struct {
 	Clouds map[string]Cloud `yaml:"clouds"`
 }
 
-// Cloud represents an entry in a clouds.yaml file.
+// Cloud represents an entry in a clouds.yaml/public-clouds.yaml/secure.yaml file.
 type Cloud struct {
+	Cloud      string        `yaml:"cloud"`
+	Profile    string        `yaml:"profile"`
 	AuthInfo   *AuthInfo     `yaml:"auth"`
 	AuthType   AuthType      `yaml:"auth_type"`
 	RegionName string        `yaml:"region_name"`
@@ -17,9 +26,24 @@ type Cloud struct {
 	// API Version overrides.
 	IdentityAPIVersion string `yaml:"identity_api_version"`
 	VolumeAPIVersion   string `yaml:"volume_api_version"`
+
+	// Verify whether or not SSL API requests should be verified.
+	Verify *bool `yaml:"verify"`
+
+	// CACertFile a path to a CA Cert bundle that can be used as part of
+	// verifying SSL API requests.
+	CACertFile string `yaml:"cacert"`
+
+	// ClientCertFile a path to a client certificate to use as part of the SSL
+	// transaction.
+	ClientCertFile string `yaml:"cert"`
+
+	// ClientKeyFile a path to a client key to use as part of the SSL
+	// transaction.
+	ClientKeyFile string `yaml:"key"`
 }
 
-// Auth represents the auth section of a cloud entry or
+// AuthInfo represents the auth section of a cloud entry or
 // auth options entered explicitly in ClientOpts.
 type AuthInfo struct {
 	// AuthURL is the keystone/identity endpoint URL.

vendor/github.com/gophercloud/utils/openstack/clientconfig/utils.go

@@ -1,14 +1,93 @@
 package clientconfig
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"os/user"
 	"path/filepath"
+	"reflect"
 )
 
-// findAndLoadYAML attempts to locate a clouds.yaml file in the following
+// defaultIfEmpty is a helper function to make it cleaner to set default value
+// for strings.
+func defaultIfEmpty(value string, defaultValue string) string {
+	if value == "" {
+		return defaultValue
+	}
+	return value
+}
+
+// mergeCLouds merges two Clouds recursively (the AuthInfo also gets merged).
+// In case both Clouds define a value, the value in the 'override' cloud takes precedence
+func mergeClouds(override, cloud interface{}) (*Cloud, error) {
+	overrideJson, err := json.Marshal(override)
+	if err != nil {
+		return nil, err
+	}
+	cloudJson, err := json.Marshal(cloud)
+	if err != nil {
+		return nil, err
+	}
+	var overrideInterface interface{}
+	err = json.Unmarshal(overrideJson, &overrideInterface)
+	if err != nil {
+		return nil, err
+	}
+	var cloudInterface interface{}
+	err = json.Unmarshal(cloudJson, &cloudInterface)
+	if err != nil {
+		return nil, err
+	}
+	var mergedCloud Cloud
+	mergedInterface := mergeInterfaces(overrideInterface, cloudInterface)
+	mergedJson, err := json.Marshal(mergedInterface)
+	json.Unmarshal(mergedJson, &mergedCloud)
+	return &mergedCloud, nil
+}
+
+// merges two interfaces. In cases where a value is defined for both 'overridingInterface' and
+// 'inferiorInterface' the value in 'overridingInterface' will take precedence.
+func mergeInterfaces(overridingInterface, inferiorInterface interface{}) interface{} {
+	switch overriding := overridingInterface.(type) {
+	case map[string]interface{}:
+		interfaceMap, ok := inferiorInterface.(map[string]interface{})
+		if !ok {
+			return overriding
+		}
+		for k, v := range interfaceMap {
+			if overridingValue, ok := overriding[k]; ok {
+				overriding[k] = mergeInterfaces(overridingValue, v)
+			} else {
+				overriding[k] = v
+			}
+		}
+	case []interface{}:
+		list, ok := inferiorInterface.([]interface{})
+		if !ok {
+			return overriding
+		}
+		for i := range list {
+			overriding = append(overriding, list[i])
+		}
+		return overriding
+	case nil:
+		// mergeClouds(nil, map[string]interface{...}) -> map[string]interface{...}
+		v, ok := inferiorInterface.(map[string]interface{})
+		if ok {
+			return v
+		}
+	}
+	// We don't want to override with empty values
+	if reflect.DeepEqual(overridingInterface, nil) || reflect.DeepEqual(reflect.Zero(reflect.TypeOf(overridingInterface)).Interface(), overridingInterface) {
+		return inferiorInterface
+	} else {
+		return overridingInterface
+	}
+}
+
+// findAndReadCloudsYAML attempts to locate a clouds.yaml file in the following
 // locations:
 //
 // 1. OS_CLIENT_CONFIG_FILE
@@ -17,7 +96,7 @@ import (
 // 4. unix-specific site_config_dir (/etc/openstack/clouds.yaml)
 //
 // If found, the contents of the file is returned.
-func findAndReadYAML() ([]byte, error) {
+func findAndReadCloudsYAML() ([]byte, error) {
 	// OS_CLIENT_CONFIG_FILE
 	if v := os.Getenv("OS_CLIENT_CONFIG_FILE"); v != "" {
 		if ok := fileExists(v); ok {
@@ -25,13 +104,25 @@ func findAndReadYAML() ([]byte, error) {
 		}
 	}
 
+	return findAndReadYAML("clouds.yaml")
+}
+
+func findAndReadPublicCloudsYAML() ([]byte, error) {
+	return findAndReadYAML("clouds-public.yaml")
+}
+
+func findAndReadSecureCloudsYAML() ([]byte, error) {
+	return findAndReadYAML("secure.yaml")
+}
+
+func findAndReadYAML(yamlFile string) ([]byte, error) {
 	// current directory
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("unable to determine working directory: %s", err)
 	}
 
-	filename := filepath.Join(cwd, "clouds.yaml")
+	filename := filepath.Join(cwd, yamlFile)
 	if ok := fileExists(filename); ok {
 		return ioutil.ReadFile(filename)
 	}
@@ -44,18 +135,18 @@ func findAndReadYAML() ([]byte, error) {
 
 	homeDir := currentUser.HomeDir
 	if homeDir != "" {
-		filename := filepath.Join(homeDir, ".config/openstack/clouds.yaml")
+		filename := filepath.Join(homeDir, ".config/openstack/"+yamlFile)
 		if ok := fileExists(filename); ok {
 			return ioutil.ReadFile(filename)
 		}
 	}
 
 	// unix-specific site config directory: /etc/openstack.
-	if ok := fileExists("/etc/openstack/clouds.yaml"); ok {
-		return ioutil.ReadFile("/etc/openstack/clouds.yaml")
+	if ok := fileExists("/etc/openstack/" + yamlFile); ok {
+		return ioutil.ReadFile("/etc/openstack/" + yamlFile)
 	}
 
-	return nil, fmt.Errorf("no clouds.yaml file found")
+	return nil, fmt.Errorf("no " + yamlFile + " file found")
 }
 
 // fileExists checks for the existence of a file at a given location.

vendor/vendor.json

@@ -816,10 +816,10 @@
 			"revisionTime": "2018-05-31T02:06:30Z"
 		},
 		{
-			"checksumSHA1": "ujo1JDey6cxwnGs4HXVCJNVrhHw=",
+			"checksumSHA1": "BYHuEArNKnTCbp/LTCwQSlaIY4Y=",
 			"path": "github.com/gophercloud/utils/openstack/clientconfig",
-			"revision": "afce78e977c56ca5407957bf67e8ecc56aab601d",
-			"revisionTime": "2018-05-22T20:53:45Z"
+			"revision": "d6e28a8b3199a79da5e74e3dde1eb878ff525f1a",
+			"revisionTime": "2018-08-06T21:57:00Z"
 		},
 		{
 			"checksumSHA1": "xSmii71kfQASGNG2C8ttmHx9KTE=",