From 1576af70cc13e1065709f4ab9db06274221e6790 Mon Sep 17 00:00:00 2001 From: Hugo Vieira Date: Fri, 3 Dec 2021 11:47:00 +0000 Subject: [PATCH] feat(config): Add env and file support to Worker Tags This feature allows the user to set the `tags` field to a string that points to either an environment variable or file. We then dynamically pull out the value and parse it into the config. This can also be done at the keys level, where you set your keys directly in Boundary's config file, but point the values to env/file. --- CHANGELOG.md | 2 + internal/cmd/config/config.go | 55 +++- internal/cmd/config/config_test.go | 286 ++++++++++++++++++ .../docs/concepts/filtering/worker-tags.mdx | 47 +++ website/content/docs/configuration/worker.mdx | 3 +- 5 files changed, 390 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d345231f5c..122b4e591f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Canonical reference for changes, improvements, and bugfixes for Boundary. ### New and Improved +* config: Add support for reading worker tags off of environment variables +as well as files. ([PR](https://github.com/hashicorp/boundary/pull/1758)) * config: Add support for go-sockaddr templates to Worker and Controller addresses. ([PR](https://github.com/hashicorp/boundary/pull/1731)) * host: Plugin-based host catalogs will now schedule updates for all diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index 824ce0523a..27f2673073 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -157,9 +158,10 @@ type Worker struct { // We use a raw interface for parsing so that people can use JSON-like // syntax that maps directly to the filter input or possibly more familiar - // key=value syntax. This is trued up in the Parse function below. - TagsRaw interface{} `hcl:"tags"` + // key=value syntax, as well as accepting a string denoting an env or file + // pointer. This is trued up in the Parse function below. Tags map[string][]string `hcl:"-"` + TagsRaw interface{} `hcl:"tags"` // StatusGracePeriod represents the period of time (as a duration) that the // worker will wait before disconnecting connections if it cannot make a @@ -337,12 +339,60 @@ func Parse(d string) (*Config, error) { if !strutil.Printable(result.Worker.Name) { return nil, errors.New("Worker name contains non-printable characters") } + if result.Worker.TagsRaw != nil { switch t := result.Worker.TagsRaw.(type) { + // We allow `tags` to be a simple string containing a URL with schema. + // See: https://github.com/hashicorp/go-secure-stdlib/blob/main/parseutil/parsepath.go + case string: + rawTags, err := parseutil.ParsePath(t) + if err != nil { + return nil, fmt.Errorf("Error parsing worker tags: %w", err) + } + + var temp []map[string]interface{} + err = hcl.Decode(&temp, rawTags) + if err != nil { + return nil, fmt.Errorf("Error decoding raw worker tags: %w", err) + } + + if err := mapstructure.WeakDecode(temp, &result.Worker.Tags); err != nil { + return nil, fmt.Errorf("Error decoding the worker's tags: %w", err) + } + // HCL allows multiple labeled blocks with the same name, turning it // into a slice of maps, hence the slice here. This format is the // one that ends up matching the JSON that we use in the expression. case []map[string]interface{}: + for _, m := range t { + for k, v := range m { + // We allow the user to pass in only the keys in HCL, and + // then set the values to point to a URL with schema. + valStr, ok := v.(string) + if !ok { + continue + } + + parsed, err := parseutil.ParsePath(valStr) + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { + return nil, fmt.Errorf("Error parsing worker tag values: %w", err) + } + if valStr == parsed { + // Nothing was found, ignore. + // WeakDecode will still parse it though as we + // don't know if this could be a valid tag. + continue + } + + var tags []string + err = json.Unmarshal([]byte(parsed), &tags) + if err != nil { + return nil, fmt.Errorf("Error unmarshalling env var/file contents: %w", err) + } + m[k] = tags + } + } + if err := mapstructure.WeakDecode(t, &result.Worker.Tags); err != nil { return nil, fmt.Errorf("Error decoding the worker's %q section: %w", "tags", err) } @@ -374,6 +424,7 @@ func Parse(d string) (*Config, error) { } } } + for k, v := range result.Worker.Tags { if k != strings.ToLower(k) { return nil, fmt.Errorf("Tag key %q is not all lower-case letters", k) diff --git a/internal/cmd/config/config_test.go b/internal/cmd/config/config_test.go index 3f93c08c91..70134e18be 100644 --- a/internal/cmd/config/config_test.go +++ b/internal/cmd/config/config_test.go @@ -337,6 +337,292 @@ func TestParsingName(t *testing.T) { } } +func TestWorkerTags(t *testing.T) { + defaultStateFn := func(t *testing.T, tags string) { + t.Setenv("BOUNDARY_WORKER_TAGS", tags) + } + tests := []struct { + name string + in string + stateFn func(t *testing.T, tags string) + actualTags string + expWorkerTags map[string][]string + expErr bool + expErrStr string + }{ + { + name: "tags in HCL", + in: ` + worker { + tags { + type = ["dev", "local"] + typetwo = "devtwo" + } + }`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo"}, + }, + expErr: false, + }, + { + name: "tags in HCL key=value", + in: ` + worker { + tags = ["type=dev", "type=local", "typetwo=devtwo"] + } + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo"}, + }, + expErr: false, + }, + { + name: "no tags", + in: ` + worker { + name = "testworker" + } + `, + expWorkerTags: nil, + expErr: false, + }, + { + name: "empty tags", + in: ` + worker { + name = "testworker" + tags = {} + } + `, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty tags 2", + in: ` + worker { + name = "testworker" + tags = [] + } + `, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty str", + in: ` + worker { + tags = "" + }`, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty env var", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "not a url - entire tags block", + in: ` + worker { + tags = "\x00" + }`, + expWorkerTags: map[string][]string{}, + expErr: true, + expErrStr: `Error parsing worker tags: error parsing url ("parse \"\\x00\": net/url: invalid control character in URL"): not a url`, + }, + { + name: "not a url - key's value set to string", + in: ` + worker { + tags { + type = "\x00" + } + } + `, + expWorkerTags: map[string][]string{ + "type": {"\x00"}, + }, + expErr: false, + }, + { + name: "one tag key", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: `type = ["dev", "local"]`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + }, + expErr: false, + }, + { + name: "multiple tag keys", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + type = ["dev", "local"] + typetwo = ["devtwo", "localtwo"] + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - entire tags block", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + { + "type": ["dev", "local"], + "typetwo": ["devtwo", "localtwo"] + } + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - keys specified in the HCL file, values point to env/file", + in: ` + worker { + tags = { + type = "env://BOUNDARY_WORKER_TAGS" + typetwo = "env://BOUNDARY_WORKER_TAGS_TWO" + } + }`, + stateFn: func(t *testing.T, tags string) { + defaultStateFn(t, tags) + t.Setenv("BOUNDARY_WORKER_TAGS_TWO", `["devtwo", "localtwo"]`) + }, + actualTags: `["dev","local"]`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - mix n' match", + in: ` + worker { + name = "web-prod-us-east-1" + tags { + type = "env://BOUNDARY_WORKER_TYPE_TAGS" + typetwo = "file://type_two_tags.json" + typethree = ["devthree", "localthree"] + } + } + `, + stateFn: func(t *testing.T, tags string) { + workerTypeTags := `["dev", "local"]` + t.Setenv("BOUNDARY_WORKER_TYPE_TAGS", workerTypeTags) + + filepath := "./type_two_tags.json" + err := os.WriteFile(filepath, []byte(`["devtwo", "localtwo"]`), 0o666) + require.NoError(t, err) + + t.Cleanup(func() { + err := os.Remove(filepath) + require.NoError(t, err) + }) + }, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + "typethree": {"devthree", "localthree"}, + }, + expErr: false, + }, + { + name: "bad json tags", + in: ` + worker { + tags = { + type = "env://BOUNDARY_WORKER_TAGS" + typetwo = "env://BOUNDARY_WORKER_TAGS" + } + }`, + stateFn: defaultStateFn, + actualTags: ` + { + "type": ["dev", "local"], + "typetwo": ["devtwo", "localtwo"] + } + `, + expWorkerTags: nil, + expErr: true, + expErrStr: "Error unmarshalling env var/file contents: json: cannot unmarshal object into Go value of type []string", + }, + { + name: "no clean mapping to internal structures", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + worker { + tags { + type = "indeed" + } + } + `, + expErr: true, + expErrStr: "Error decoding the worker's tags: 1 error(s) decoding:\n\n* '[0][worker][0]' expected type 'string', got unconvertible type 'map[string]interface {}', value: 'map[tags:[map[type:indeed]]]'", + }, + { + name: "not HCL", + in: `worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: `not_hcl`, + expErr: true, + expErrStr: "Error decoding raw worker tags: At 1:9: key 'not_hcl' expected start of object ('{') or assignment ('=')", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.stateFn != nil { + tt.stateFn(t, tt.actualTags) + } + + c, err := Parse(tt.in) + if tt.expErr { + require.EqualError(t, err, tt.expErrStr) + require.Nil(t, c) + return + } + + require.NoError(t, err) + require.NotNil(t, c) + require.NotNil(t, c.Worker) + require.Equal(t, tt.expWorkerTags, c.Worker.Tags) + }) + } +} + func TestController_EventingConfig(t *testing.T) { t.Parallel() diff --git a/website/content/docs/concepts/filtering/worker-tags.mdx b/website/content/docs/concepts/filtering/worker-tags.mdx index dcc325508f..d0d6267268 100644 --- a/website/content/docs/concepts/filtering/worker-tags.mdx +++ b/website/content/docs/concepts/filtering/worker-tags.mdx @@ -53,6 +53,53 @@ worker { In this format, it is not possible to have an equal sign be a part of the key. +It is also possible to set the entire `tags` block or the keys' values within +to point to an environment variable or filepath in the system, through the +`env://` and `file://` URLs: + +```hcl +worker { + name = "web-prod-us-east-1" + tags = "env://BOUNDARY_ALL_WORKER_TAGS" +} +``` + +```hcl +worker { + name = "web-prod-us-east-1" + tags { + type = "env://BOUNDARY_WORKER_TYPE_TAGS" + region = "file://config/worker/region_tags" + usage = ["admin"] + } +} +``` + +Note that the syntax within the environment variable / file changes +slightly depending on how the configuration file is set: + +For setting the entire `tags` block, both the keys and values need +to be specified, in JSON or HCL format: + +```json +{ + "region": ["us-east-1"], + "type": ["prod", "webservers"] +} +``` + +```hcl +region = ["us-east-1"] +type = ["prod", "webservers"] +``` + +For setting the keys' values within the `tags` block, only a JSON +array with the tags intended for the particular key is required: + +```json +["prod", "webservers"] +``` + # Target Worker Filtering Once workers have tags, it is possible to use these tags to control which diff --git a/website/content/docs/configuration/worker.mdx b/website/content/docs/configuration/worker.mdx index 7a371faf03..93e1e6d172 100644 --- a/website/content/docs/configuration/worker.mdx +++ b/website/content/docs/configuration/worker.mdx @@ -39,7 +39,8 @@ worker { - `tags` - A map of key-value pairs where values are an array of strings. Most commonly used for [filtering](/docs/concepts/filtering) targets a worker can proxy via [worker tags](/docs/concepts/filtering/worker-tags). On `SIGHUP`, the - tags set here will be re-parsed and new values used.. + tags set here will be re-parsed and new values used. It can also be a string + referring to a file on disk (file://) or an env var (env://). ## KMS Configuration