From d8793e3f856cb92e27394d922c4d9b2fc01b0748 Mon Sep 17 00:00:00 2001 From: Brendan Devenney Date: Sat, 23 Feb 2019 00:09:07 +0000 Subject: [PATCH 1/4] :sparkles: Store comments in the Template structure Signed-off-by: Brendan Devenney --- template/parse.go | 23 +++++++++++++++++++++++ template/parse_test.go | 3 +++ template/template.go | 1 + 3 files changed, 27 insertions(+) diff --git a/template/parse.go b/template/parse.go index 9f37929a3..722b1fb3c 100644 --- a/template/parse.go +++ b/template/parse.go @@ -24,6 +24,7 @@ type rawTemplate struct { Description string Builders []map[string]interface{} + Comments map[string]string Push map[string]interface{} PostProcessors []interface{} `mapstructure:"post-processors"` Provisioners []map[string]interface{} @@ -44,6 +45,15 @@ func (r *rawTemplate) Template() (*Template, error) { result.MinVersion = r.MinVersion result.RawContents = r.RawContents + // Gather the comments + if len(r.Comments) > 0 { + result.Comments = make(map[string]string, len(r.Comments)) + + for k, v := range r.Comments { + result.Comments[k] = v + } + } + // Gather the variables if len(r.Variables) > 0 { result.Variables = make(map[string]*Variable, len(r.Variables)) @@ -304,9 +314,22 @@ func Parse(r io.Reader) (*Template, error) { // Build an error if there are unused root level keys if len(md.Unused) > 0 { sort.Strings(md.Unused) + + unusedMap, ok := raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("Failed to convert unused root level keys to map") + } + for _, unused := range md.Unused { // Ignore keys starting with '_' as comments if unused[0] == '_' { + if rawTpl.Comments == nil { + rawTpl.Comments = make(map[string]string) + } + rawTpl.Comments[unused], ok = unusedMap[unused].(string) + if !ok { + return nil, fmt.Errorf("Failed to cast root level comment key to string") + } continue } diff --git a/template/parse_test.go b/template/parse_test.go index 8881783f6..5bb13ea4b 100644 --- a/template/parse_test.go +++ b/template/parse_test.go @@ -316,6 +316,9 @@ func TestParse(t *testing.T) { Type: "something", }, }, + Comments: map[string]string{ + "_info": "foo", + }, }, false, }, diff --git a/template/template.go b/template/template.go index 3a0a372aa..635e9c202 100644 --- a/template/template.go +++ b/template/template.go @@ -18,6 +18,7 @@ type Template struct { Description string MinVersion string + Comments map[string]string Variables map[string]*Variable SensitiveVariables []*Variable Builders map[string]*Builder From 610eecfc9911ed5fe4be9a33a391968c9511743d Mon Sep 17 00:00:00 2001 From: Brendan Devenney Date: Sat, 23 Feb 2019 04:52:03 +0000 Subject: [PATCH 2/4] :sparkles: Track sensitive variable keys to support JSON template writing Signed-off-by: Brendan Devenney --- template/parse.go | 1 + template/template.go | 1 + 2 files changed, 2 insertions(+) diff --git a/template/parse.go b/template/parse.go index 722b1fb3c..23ce3393a 100644 --- a/template/parse.go +++ b/template/parse.go @@ -60,6 +60,7 @@ func (r *rawTemplate) Template() (*Template, error) { } for k, rawV := range r.Variables { var v Variable + v.Key = k // Variable is required if the value is exactly nil v.Required = rawV == nil diff --git a/template/template.go b/template/template.go index 635e9c202..ffbde7ac9 100644 --- a/template/template.go +++ b/template/template.go @@ -70,6 +70,7 @@ type Push struct { // Variable represents a variable within the template type Variable struct { + Key string Default string Required bool } From afba444373c3a4aa7af727ae83954dd94acbcc3f Mon Sep 17 00:00:00 2001 From: Brendan Devenney Date: Sat, 23 Feb 2019 21:41:53 +0000 Subject: [PATCH 3/4] :sparkles: Refactor rawTemplate to better align with real raw template structure Signed-off-by: Brendan Devenney --- template/parse.go | 68 ++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/template/parse.go b/template/parse.go index 23ce3393a..885884ede 100644 --- a/template/parse.go +++ b/template/parse.go @@ -23,11 +23,11 @@ type rawTemplate struct { MinVersion string `mapstructure:"min_packer_version"` Description string - Builders []map[string]interface{} - Comments map[string]string + Builders []interface{} `mapstructure:"builders"` + Comments []map[string]string Push map[string]interface{} PostProcessors []interface{} `mapstructure:"post-processors"` - Provisioners []map[string]interface{} + Provisioners []interface{} Variables map[string]interface{} SensitiveVariables []string `mapstructure:"sensitive-variables"` @@ -49,8 +49,10 @@ func (r *rawTemplate) Template() (*Template, error) { if len(r.Comments) > 0 { result.Comments = make(map[string]string, len(r.Comments)) - for k, v := range r.Comments { - result.Comments[k] = v + for _, c := range r.Comments { + for k, v := range c { + result.Comments[k] = v + } } } @@ -94,9 +96,11 @@ func (r *rawTemplate) Template() (*Template, error) { } // Set the raw configuration and delete any special keys - b.Config = rawB + b.Config = rawB.(map[string]interface{}) + delete(b.Config, "name") delete(b.Config, "type") + if len(b.Config) == 0 { b.Config = nil } @@ -155,14 +159,17 @@ func (r *rawTemplate) Template() (*Template, error) { continue } - // Set the configuration - delete(c, "except") - delete(c, "only") - delete(c, "keep_input_artifact") - delete(c, "type") - delete(c, "name") - if len(c) > 0 { - pp.Config = c + // Set the raw configuration and delete any special keys + pp.Config = c + + delete(pp.Config, "except") + delete(pp.Config, "only") + delete(pp.Config, "keep_input_artifact") + delete(pp.Config, "type") + delete(pp.Config, "name") + + if len(pp.Config) == 0 { + pp.Config = nil } pps = append(pps, &pp) @@ -190,17 +197,19 @@ func (r *rawTemplate) Template() (*Template, error) { continue } - // Copy the configuration - delete(v, "except") - delete(v, "only") - delete(v, "override") - delete(v, "pause_before") - delete(v, "type") - if len(v) > 0 { - p.Config = v + // Set the raw configuration and delete any special keys + p.Config = v.(map[string]interface{}) + + delete(p.Config, "except") + delete(p.Config, "only") + delete(p.Config, "override") + delete(p.Config, "pause_before") + delete(p.Config, "type") + + if len(p.Config) == 0 { + p.Config = nil } - // TODO: stuff result.Provisioners = append(result.Provisioners, &p) } @@ -324,13 +333,16 @@ func Parse(r io.Reader) (*Template, error) { for _, unused := range md.Unused { // Ignore keys starting with '_' as comments if unused[0] == '_' { - if rawTpl.Comments == nil { - rawTpl.Comments = make(map[string]string) - } - rawTpl.Comments[unused], ok = unusedMap[unused].(string) + commentVal, ok := unusedMap[unused].(string) if !ok { - return nil, fmt.Errorf("Failed to cast root level comment key to string") + return nil, fmt.Errorf("Failed to cast root level comment value to string") } + + comment := map[string]string{ + unused: commentVal, + } + + rawTpl.Comments = append(rawTpl.Comments, comment) continue } From 4d2a5fb9a2fa65ae70b6d721f81ff73c8b394b92 Mon Sep 17 00:00:00 2001 From: Brendan Devenney Date: Sat, 23 Feb 2019 22:38:04 +0000 Subject: [PATCH 4/4] :sparkles: Implement template marshalling logic Signed-off-by: Brendan Devenney --- template/parse.go | 48 +++-- template/parse_test.go | 170 +++++++++++++++++- template/template.go | 153 ++++++++++++++-- .../test-fixtures/parse-basic-config.json | 3 + template/test-fixtures/parse-monolithic.json | 61 +++++++ .../parse-provisioner-config.json | 5 + 6 files changed, 409 insertions(+), 31 deletions(-) create mode 100644 template/test-fixtures/parse-basic-config.json create mode 100644 template/test-fixtures/parse-monolithic.json create mode 100644 template/test-fixtures/parse-provisioner-config.json diff --git a/template/parse.go b/template/parse.go index 885884ede..bc18a02e4 100644 --- a/template/parse.go +++ b/template/parse.go @@ -20,18 +20,40 @@ import ( // This is what is decoded directly from the file, and then it is turned // into a Template object thereafter. type rawTemplate struct { - MinVersion string `mapstructure:"min_packer_version"` - Description string - - Builders []interface{} `mapstructure:"builders"` - Comments []map[string]string - Push map[string]interface{} - PostProcessors []interface{} `mapstructure:"post-processors"` - Provisioners []interface{} - Variables map[string]interface{} - SensitiveVariables []string `mapstructure:"sensitive-variables"` - - RawContents []byte + MinVersion string `mapstructure:"min_packer_version" json:"min_packer_version,omitempty"` + Description string `json:"description,omitempty"` + + Builders []interface{} `mapstructure:"builders" json:"builders,omitempty"` + Comments []map[string]string `json:"comments,omitempty"` + Push map[string]interface{} `json:"push,omitempty"` + PostProcessors []interface{} `mapstructure:"post-processors" json:"post-processors,omitempty"` + Provisioners []interface{} `json:"provisioners,omitempty"` + Variables map[string]interface{} `json:"variables,omitempty"` + SensitiveVariables []string `mapstructure:"sensitive-variables" json:"sensitive-variables,omitempty"` + + RawContents []byte `json:"-"` +} + +// MarshalJSON conducts the necessary flattening of the rawTemplate struct +// to provide valid Packer template JSON +func (r *rawTemplate) MarshalJSON() ([]byte, error) { + // Avoid recursion + type rawTemplate_ rawTemplate + out, _ := json.Marshal(rawTemplate_(*r)) + + var m map[string]json.RawMessage + _ = json.Unmarshal(out, &m) + + // Flatten Comments + delete(m, "comments") + for _, comment := range r.Comments { + for k, v := range comment { + out, _ = json.Marshal(v) + m[k] = out + } + } + + return json.Marshal(m) } // Template returns the actual Template object built from this raw @@ -60,6 +82,7 @@ func (r *rawTemplate) Template() (*Template, error) { if len(r.Variables) > 0 { result.Variables = make(map[string]*Variable, len(r.Variables)) } + for k, rawV := range r.Variables { var v Variable v.Key = k @@ -331,7 +354,6 @@ func Parse(r io.Reader) (*Template, error) { } for _, unused := range md.Unused { - // Ignore keys starting with '_' as comments if unused[0] == '_' { commentVal, ok := unusedMap[unused].(string) if !ok { diff --git a/template/parse_test.go b/template/parse_test.go index 5bb13ea4b..6bed10e14 100644 --- a/template/parse_test.go +++ b/template/parse_test.go @@ -3,6 +3,8 @@ package template import ( + "bytes" + "encoding/json" "path/filepath" "reflect" "strings" @@ -31,6 +33,21 @@ func TestParse(t *testing.T) { }, false, }, + { + "parse-basic-config.json", + &Template{ + Builders: map[string]*Builder{ + "something": { + Name: "something", + Type: "something", + Config: map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + false, + }, { "parse-builder-no-type.json", nil, @@ -56,7 +73,20 @@ func TestParse(t *testing.T) { }, false, }, - + { + "parse-provisioner-config.json", + &Template{ + Provisioners: []*Provisioner{ + { + Type: "something", + Config: map[string]interface{}{ + "inline": "echo 'foo'", + }, + }, + }, + }, + false, + }, { "parse-provisioner-pause-before.json", &Template{ @@ -127,6 +157,7 @@ func TestParse(t *testing.T) { Variables: map[string]*Variable{ "foo": { Default: "foo", + Key: "foo", }, }, }, @@ -139,6 +170,7 @@ func TestParse(t *testing.T) { Variables: map[string]*Variable{ "foo": { Required: true, + Key: "foo", }, }, }, @@ -306,7 +338,6 @@ func TestParse(t *testing.T) { }, false, }, - { "parse-comment.json", &Template{ @@ -322,13 +353,114 @@ func TestParse(t *testing.T) { }, false, }, + { + "parse-monolithic.json", + &Template{ + Comments: map[string]string{ + "_comment": "comment", + }, + Description: "Description Test", + MinVersion: "1.3.0", + SensitiveVariables: []*Variable{ + { + Required: false, + Key: "one", + Default: "1", + }, + }, + Variables: map[string]*Variable{ + "one": { + Required: false, + Key: "one", + Default: "1", + }, + "two": { + Required: false, + Key: "two", + Default: "2", + }, + "three": { + Required: true, + Key: "three", + Default: "", + }, + }, + Builders: map[string]*Builder{ + "amazon-ebs": { + Name: "amazon-ebs", + Type: "amazon-ebs", + Config: map[string]interface{}{ + "ami_name": "AMI Name", + "instance_type": "t2.micro", + "ssh_username": "ec2-user", + "source_ami": "ami-aaaaaaaaaaaaaa", + }, + }, + "docker": { + Name: "docker", + Type: "docker", + Config: map[string]interface{}{ + "image": "ubuntu", + "export_path": "image.tar", + }, + }, + }, + Provisioners: []*Provisioner{ + { + Type: "shell", + Config: map[string]interface{}{ + "script": "script.sh", + }, + }, + { + Type: "shell", + Config: map[string]interface{}{ + "script": "script.sh", + }, + Override: map[string]interface{}{ + "docker": map[string]interface{}{ + "execute_command": "echo 'override'", + }, + }, + }, + }, + PostProcessors: [][]*PostProcessor{ + { + { + Type: "compress", + }, + { + Type: "vagrant", + OnlyExcept: OnlyExcept{ + Only: []string{"docker"}, + }, + }, + }, + { + { + Type: "shell-local", + Config: map[string]interface{}{ + "inline": []interface{}{"echo foo"}, + }, + OnlyExcept: OnlyExcept{ + Except: []string{"amazon-ebs"}, + }, + }, + }, + }, + Push: Push{ + Name: "push test", + }, + }, + false, + }, } for _, tc := range cases { path, _ := filepath.Abs(fixtureDir(tc.File)) tpl, err := ParseFile(fixtureDir(tc.File)) if (err != nil) != tc.Err { - t.Fatalf("err: %s", err) + t.Fatalf("%s\n\nerr: %s", tc.File, err) } if tc.Result != nil { @@ -340,6 +472,38 @@ func TestParse(t *testing.T) { if !reflect.DeepEqual(tpl, tc.Result) { t.Fatalf("bad: %s\n\n%#v\n\n%#v", tc.File, tpl, tc.Result) } + + // Only test template writing if the template is valid + if tc.Err == false { + // Get rawTemplate + raw, err := tpl.Raw() + if err != nil { + t.Fatalf("Failed to convert back to raw template: %s\n\n%v\n\n%s", tc.File, tpl, err) + } + + out, _ := json.MarshalIndent(raw, "", " ") + if err != nil { + t.Fatalf("Failed to marshal raw template: %s\n\n%v\n\n%s", tc.File, raw, err) + } + + // Write JSON to a buffer (emulates filesystem write without dirtying the workspace) + fileBuf := bytes.NewBuffer(out) + + // Parse the JSON template we wrote to our buffer + tplRewritten, err := Parse(fileBuf) + if err != nil { + t.Fatalf("Failed to re-read marshalled template: %s\n\n%v\n\n%s\n\n%s", tc.File, tpl, out, err) + } + + // Override the metadata we don't care about (file path, raw file contents) + tplRewritten.Path = path + tplRewritten.RawContents = nil + + // Test that our output raw template is functionally equal + if !reflect.DeepEqual(tpl, tplRewritten) { + t.Fatalf("Data lost when writing template to file: %s\n\n%v\n\n%v\n\n%s", tc.File, tpl, tplRewritten, out) + } + } } } diff --git a/template/template.go b/template/template.go index ffbde7ac9..6f978a5db 100644 --- a/template/template.go +++ b/template/template.go @@ -1,6 +1,7 @@ package template import ( + "encoding/json" "errors" "fmt" "time" @@ -30,31 +31,143 @@ type Template struct { RawContents []byte } +// Raw converts a Template struct back into the raw Packer template structure +func (t *Template) Raw() (*rawTemplate, error) { + var out rawTemplate + + out.MinVersion = t.MinVersion + out.Description = t.Description + + for k, v := range t.Comments { + out.Comments = append(out.Comments, map[string]string{k: v}) + } + + for _, b := range t.Builders { + out.Builders = append(out.Builders, b) + } + + for _, p := range t.Provisioners { + out.Provisioners = append(out.Provisioners, p) + } + + for _, pp := range t.PostProcessors { + out.PostProcessors = append(out.PostProcessors, pp) + } + + for _, v := range t.SensitiveVariables { + out.SensitiveVariables = append(out.SensitiveVariables, v.Key) + } + + for k, v := range t.Variables { + if out.Variables == nil { + out.Variables = make(map[string]interface{}) + } + + out.Variables[k] = v + } + + if t.Push.Name != "" { + b, _ := json.Marshal(t.Push) + + var m map[string]interface{} + _ = json.Unmarshal(b, &m) + + out.Push = m + } + + return &out, nil +} + // Builder represents a builder configured in the template type Builder struct { - Name string - Type string - Config map[string]interface{} + Name string `json:"name,omitempty"` + Type string `json:"type"` + Config map[string]interface{} `json:"config,omitempty"` +} + +// MarshalJSON conducts the necessary flattening of the Builder struct +// to provide valid Packer template JSON +func (b *Builder) MarshalJSON() ([]byte, error) { + // Avoid recursion + type Builder_ Builder + out, _ := json.Marshal(Builder_(*b)) + + var m map[string]json.RawMessage + _ = json.Unmarshal(out, &m) + + // Flatten Config + delete(m, "config") + for k, v := range b.Config { + out, _ = json.Marshal(v) + m[k] = out + } + + return json.Marshal(m) } // PostProcessor represents a post-processor within the template. type PostProcessor struct { - OnlyExcept `mapstructure:",squash"` + OnlyExcept `mapstructure:",squash" json:",omitempty"` - Name string - Type string - KeepInputArtifact bool `mapstructure:"keep_input_artifact"` - Config map[string]interface{} + Name string `json:"name,omitempty"` + Type string `json:"type"` + KeepInputArtifact bool `mapstructure:"keep_input_artifact" json:"keep_input_artifact,omitempty"` + Config map[string]interface{} `json:"config,omitempty"` +} + +// MarshalJSON conducts the necessary flattening of the PostProcessor struct +// to provide valid Packer template JSON +func (p *PostProcessor) MarshalJSON() ([]byte, error) { + // Early exit for simple definitions + if len(p.Config) == 0 && len(p.OnlyExcept.Only) == 0 && len(p.OnlyExcept.Except) == 0 && !p.KeepInputArtifact { + return json.Marshal(p.Type) + } + + // Avoid recursion + type PostProcessor_ PostProcessor + out, _ := json.Marshal(PostProcessor_(*p)) + + var m map[string]json.RawMessage + _ = json.Unmarshal(out, &m) + + // Flatten Config + delete(m, "config") + for k, v := range p.Config { + out, _ = json.Marshal(v) + m[k] = out + } + + return json.Marshal(m) } // Provisioner represents a provisioner within the template. type Provisioner struct { - OnlyExcept `mapstructure:",squash"` + OnlyExcept `mapstructure:",squash" json:",omitempty"` + + Type string `json:"type"` + Config map[string]interface{} `json:"config,omitempty"` + Override map[string]interface{} `json:"override,omitempty"` + PauseBefore time.Duration `mapstructure:"pause_before" json:"pause_before,omitempty"` +} + +// MarshalJSON conducts the necessary flattening of the Provisioner struct +// to provide valid Packer template JSON +func (p *Provisioner) MarshalJSON() ([]byte, error) { + // Avoid recursion + type Provisioner_ Provisioner + out, _ := json.Marshal(Provisioner_(*p)) + + var m map[string]json.RawMessage + _ = json.Unmarshal(out, &m) + + // Flatten Config + delete(m, "config") + for k, v := range p.Config { + out, _ = json.Marshal(v) + m[k] = out + } - Type string - Config map[string]interface{} - Override map[string]interface{} - PauseBefore time.Duration `mapstructure:"pause_before"` + return json.Marshal(m) } // Push represents the configuration for pushing the template to Atlas. @@ -75,11 +188,21 @@ type Variable struct { Required bool } +func (v *Variable) MarshalJSON() ([]byte, error) { + if v.Required { + // We use a nil pointer to coax Go into marshalling it as a JSON null + var ret *string + return json.Marshal(ret) + } + + return json.Marshal(v.Default) +} + // OnlyExcept is a struct that is meant to be embedded that contains the // logic required for "only" and "except" meta-parameters. type OnlyExcept struct { - Only []string - Except []string + Only []string `json:"only,omitempty"` + Except []string `json:"except,omitempty"` } //------------------------------------------------------------------- diff --git a/template/test-fixtures/parse-basic-config.json b/template/test-fixtures/parse-basic-config.json new file mode 100644 index 000000000..ee7695bd9 --- /dev/null +++ b/template/test-fixtures/parse-basic-config.json @@ -0,0 +1,3 @@ +{ + "builders": [{"type": "something", "foo": "bar"}] +} diff --git a/template/test-fixtures/parse-monolithic.json b/template/test-fixtures/parse-monolithic.json new file mode 100644 index 000000000..a04f1392d --- /dev/null +++ b/template/test-fixtures/parse-monolithic.json @@ -0,0 +1,61 @@ +{ + "_comment": "comment", + "description": "Description Test", + "min_packer_version": "1.3.0", + "variables": { + "one": "1", + "two": "2", + "three": null + }, + "sensitive-variables": ["one"], + "builders": [ + { + "type": "amazon-ebs", + + "ami_name": "AMI Name", + "instance_type": "t2.micro", + "ssh_username": "ec2-user", + "source_ami": "ami-aaaaaaaaaaaaaa" + }, + { + "type": "docker", + + "image": "ubuntu", + "export_path": "image.tar" + } + ], + "provisioners": [ + { + "type": "shell", + "script": "script.sh" + }, + { + "type": "shell", + "script": "script.sh", + "override": { + "docker": { + "execute_command": "echo 'override'" + } + } + } + ], + "post-processors": [ + [ + "compress", + { + "type": "vagrant", + "only": ["docker"] + } + ], + [ + { + "type": "shell-local", + "inline": ["echo foo"], + "except": ["amazon-ebs"] + } + ] + ], + "push": { + "name": "push test" + } +} diff --git a/template/test-fixtures/parse-provisioner-config.json b/template/test-fixtures/parse-provisioner-config.json new file mode 100644 index 000000000..24cd0fe93 --- /dev/null +++ b/template/test-fixtures/parse-provisioner-config.json @@ -0,0 +1,5 @@ +{ + "provisioners": [ + {"type": "something","inline":"echo 'foo'"} + ] +}