diff --git a/CHANGELOG.md b/CHANGELOG.md index d345231f5c..122b4e591f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Canonical reference for changes, improvements, and bugfixes for Boundary. ### New and Improved +* config: Add support for reading worker tags off of environment variables +as well as files. ([PR](https://github.com/hashicorp/boundary/pull/1758)) * config: Add support for go-sockaddr templates to Worker and Controller addresses. ([PR](https://github.com/hashicorp/boundary/pull/1731)) * host: Plugin-based host catalogs will now schedule updates for all diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index 824ce0523a..27f2673073 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -157,9 +158,10 @@ type Worker struct { // We use a raw interface for parsing so that people can use JSON-like // syntax that maps directly to the filter input or possibly more familiar - // key=value syntax. This is trued up in the Parse function below. - TagsRaw interface{} `hcl:"tags"` + // key=value syntax, as well as accepting a string denoting an env or file + // pointer. This is trued up in the Parse function below. Tags map[string][]string `hcl:"-"` + TagsRaw interface{} `hcl:"tags"` // StatusGracePeriod represents the period of time (as a duration) that the // worker will wait before disconnecting connections if it cannot make a @@ -337,12 +339,60 @@ func Parse(d string) (*Config, error) { if !strutil.Printable(result.Worker.Name) { return nil, errors.New("Worker name contains non-printable characters") } + if result.Worker.TagsRaw != nil { switch t := result.Worker.TagsRaw.(type) { + // We allow `tags` to be a simple string containing a URL with schema. + // See: https://github.com/hashicorp/go-secure-stdlib/blob/main/parseutil/parsepath.go + case string: + rawTags, err := parseutil.ParsePath(t) + if err != nil { + return nil, fmt.Errorf("Error parsing worker tags: %w", err) + } + + var temp []map[string]interface{} + err = hcl.Decode(&temp, rawTags) + if err != nil { + return nil, fmt.Errorf("Error decoding raw worker tags: %w", err) + } + + if err := mapstructure.WeakDecode(temp, &result.Worker.Tags); err != nil { + return nil, fmt.Errorf("Error decoding the worker's tags: %w", err) + } + // HCL allows multiple labeled blocks with the same name, turning it // into a slice of maps, hence the slice here. This format is the // one that ends up matching the JSON that we use in the expression. case []map[string]interface{}: + for _, m := range t { + for k, v := range m { + // We allow the user to pass in only the keys in HCL, and + // then set the values to point to a URL with schema. + valStr, ok := v.(string) + if !ok { + continue + } + + parsed, err := parseutil.ParsePath(valStr) + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { + return nil, fmt.Errorf("Error parsing worker tag values: %w", err) + } + if valStr == parsed { + // Nothing was found, ignore. + // WeakDecode will still parse it though as we + // don't know if this could be a valid tag. + continue + } + + var tags []string + err = json.Unmarshal([]byte(parsed), &tags) + if err != nil { + return nil, fmt.Errorf("Error unmarshalling env var/file contents: %w", err) + } + m[k] = tags + } + } + if err := mapstructure.WeakDecode(t, &result.Worker.Tags); err != nil { return nil, fmt.Errorf("Error decoding the worker's %q section: %w", "tags", err) } @@ -374,6 +424,7 @@ func Parse(d string) (*Config, error) { } } } + for k, v := range result.Worker.Tags { if k != strings.ToLower(k) { return nil, fmt.Errorf("Tag key %q is not all lower-case letters", k) diff --git a/internal/cmd/config/config_test.go b/internal/cmd/config/config_test.go index 3f93c08c91..70134e18be 100644 --- a/internal/cmd/config/config_test.go +++ b/internal/cmd/config/config_test.go @@ -337,6 +337,292 @@ func TestParsingName(t *testing.T) { } } +func TestWorkerTags(t *testing.T) { + defaultStateFn := func(t *testing.T, tags string) { + t.Setenv("BOUNDARY_WORKER_TAGS", tags) + } + tests := []struct { + name string + in string + stateFn func(t *testing.T, tags string) + actualTags string + expWorkerTags map[string][]string + expErr bool + expErrStr string + }{ + { + name: "tags in HCL", + in: ` + worker { + tags { + type = ["dev", "local"] + typetwo = "devtwo" + } + }`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo"}, + }, + expErr: false, + }, + { + name: "tags in HCL key=value", + in: ` + worker { + tags = ["type=dev", "type=local", "typetwo=devtwo"] + } + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo"}, + }, + expErr: false, + }, + { + name: "no tags", + in: ` + worker { + name = "testworker" + } + `, + expWorkerTags: nil, + expErr: false, + }, + { + name: "empty tags", + in: ` + worker { + name = "testworker" + tags = {} + } + `, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty tags 2", + in: ` + worker { + name = "testworker" + tags = [] + } + `, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty str", + in: ` + worker { + tags = "" + }`, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "empty env var", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + expWorkerTags: map[string][]string{}, + expErr: false, + }, + { + name: "not a url - entire tags block", + in: ` + worker { + tags = "\x00" + }`, + expWorkerTags: map[string][]string{}, + expErr: true, + expErrStr: `Error parsing worker tags: error parsing url ("parse \"\\x00\": net/url: invalid control character in URL"): not a url`, + }, + { + name: "not a url - key's value set to string", + in: ` + worker { + tags { + type = "\x00" + } + } + `, + expWorkerTags: map[string][]string{ + "type": {"\x00"}, + }, + expErr: false, + }, + { + name: "one tag key", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: `type = ["dev", "local"]`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + }, + expErr: false, + }, + { + name: "multiple tag keys", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + type = ["dev", "local"] + typetwo = ["devtwo", "localtwo"] + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - entire tags block", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + { + "type": ["dev", "local"], + "typetwo": ["devtwo", "localtwo"] + } + `, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - keys specified in the HCL file, values point to env/file", + in: ` + worker { + tags = { + type = "env://BOUNDARY_WORKER_TAGS" + typetwo = "env://BOUNDARY_WORKER_TAGS_TWO" + } + }`, + stateFn: func(t *testing.T, tags string) { + defaultStateFn(t, tags) + t.Setenv("BOUNDARY_WORKER_TAGS_TWO", `["devtwo", "localtwo"]`) + }, + actualTags: `["dev","local"]`, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + }, + expErr: false, + }, + { + name: "json tags - mix n' match", + in: ` + worker { + name = "web-prod-us-east-1" + tags { + type = "env://BOUNDARY_WORKER_TYPE_TAGS" + typetwo = "file://type_two_tags.json" + typethree = ["devthree", "localthree"] + } + } + `, + stateFn: func(t *testing.T, tags string) { + workerTypeTags := `["dev", "local"]` + t.Setenv("BOUNDARY_WORKER_TYPE_TAGS", workerTypeTags) + + filepath := "./type_two_tags.json" + err := os.WriteFile(filepath, []byte(`["devtwo", "localtwo"]`), 0o666) + require.NoError(t, err) + + t.Cleanup(func() { + err := os.Remove(filepath) + require.NoError(t, err) + }) + }, + expWorkerTags: map[string][]string{ + "type": {"dev", "local"}, + "typetwo": {"devtwo", "localtwo"}, + "typethree": {"devthree", "localthree"}, + }, + expErr: false, + }, + { + name: "bad json tags", + in: ` + worker { + tags = { + type = "env://BOUNDARY_WORKER_TAGS" + typetwo = "env://BOUNDARY_WORKER_TAGS" + } + }`, + stateFn: defaultStateFn, + actualTags: ` + { + "type": ["dev", "local"], + "typetwo": ["devtwo", "localtwo"] + } + `, + expWorkerTags: nil, + expErr: true, + expErrStr: "Error unmarshalling env var/file contents: json: cannot unmarshal object into Go value of type []string", + }, + { + name: "no clean mapping to internal structures", + in: ` + worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: ` + worker { + tags { + type = "indeed" + } + } + `, + expErr: true, + expErrStr: "Error decoding the worker's tags: 1 error(s) decoding:\n\n* '[0][worker][0]' expected type 'string', got unconvertible type 'map[string]interface {}', value: 'map[tags:[map[type:indeed]]]'", + }, + { + name: "not HCL", + in: `worker { + tags = "env://BOUNDARY_WORKER_TAGS" + }`, + stateFn: defaultStateFn, + actualTags: `not_hcl`, + expErr: true, + expErrStr: "Error decoding raw worker tags: At 1:9: key 'not_hcl' expected start of object ('{') or assignment ('=')", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.stateFn != nil { + tt.stateFn(t, tt.actualTags) + } + + c, err := Parse(tt.in) + if tt.expErr { + require.EqualError(t, err, tt.expErrStr) + require.Nil(t, c) + return + } + + require.NoError(t, err) + require.NotNil(t, c) + require.NotNil(t, c.Worker) + require.Equal(t, tt.expWorkerTags, c.Worker.Tags) + }) + } +} + func TestController_EventingConfig(t *testing.T) { t.Parallel() diff --git a/website/content/docs/concepts/filtering/worker-tags.mdx b/website/content/docs/concepts/filtering/worker-tags.mdx index dcc325508f..d0d6267268 100644 --- a/website/content/docs/concepts/filtering/worker-tags.mdx +++ b/website/content/docs/concepts/filtering/worker-tags.mdx @@ -53,6 +53,53 @@ worker { In this format, it is not possible to have an equal sign be a part of the key. +It is also possible to set the entire `tags` block or the keys' values within +to point to an environment variable or filepath in the system, through the +`env://` and `file://` URLs: + +```hcl +worker { + name = "web-prod-us-east-1" + tags = "env://BOUNDARY_ALL_WORKER_TAGS" +} +``` + +```hcl +worker { + name = "web-prod-us-east-1" + tags { + type = "env://BOUNDARY_WORKER_TYPE_TAGS" + region = "file://config/worker/region_tags" + usage = ["admin"] + } +} +``` + +Note that the syntax within the environment variable / file changes +slightly depending on how the configuration file is set: + +For setting the entire `tags` block, both the keys and values need +to be specified, in JSON or HCL format: + +```json +{ + "region": ["us-east-1"], + "type": ["prod", "webservers"] +} +``` + +```hcl +region = ["us-east-1"] +type = ["prod", "webservers"] +``` + +For setting the keys' values within the `tags` block, only a JSON +array with the tags intended for the particular key is required: + +```json +["prod", "webservers"] +``` + # Target Worker Filtering Once workers have tags, it is possible to use these tags to control which diff --git a/website/content/docs/configuration/worker.mdx b/website/content/docs/configuration/worker.mdx index 7a371faf03..93e1e6d172 100644 --- a/website/content/docs/configuration/worker.mdx +++ b/website/content/docs/configuration/worker.mdx @@ -39,7 +39,8 @@ worker { - `tags` - A map of key-value pairs where values are an array of strings. Most commonly used for [filtering](/docs/concepts/filtering) targets a worker can proxy via [worker tags](/docs/concepts/filtering/worker-tags). On `SIGHUP`, the - tags set here will be re-parsed and new values used.. + tags set here will be re-parsed and new values used. It can also be a string + referring to a file on disk (file://) or an env var (env://). ## KMS Configuration