From aff7d360e1fcd340ef356694eede9c565e1b1a33 Mon Sep 17 00:00:00 2001 From: Liam Cervante Date: Mon, 9 Jan 2023 11:05:25 +0100 Subject: [PATCH] Add skeleton functions and API for refactored renderer (#32368) * prep for processing the structured run output * undo unwanted change to a json key * Add skeleton functions and API for refactored renderer * goimports * Fix documentation of the RenderOpts struct * Add README explaining implementation details for renderer and plans for future expansion * Update internal/command/jsonformat/README.md Co-authored-by: Alisdair McDiarmid * address comments Co-authored-by: Alisdair McDiarmid --- internal/command/jsonformat/README.md | 247 ++++++++++++++++++ internal/command/jsonformat/change/change.go | 57 ++++ .../command/jsonformat/change/renderer.go | 34 +++ .../command/jsonformat/differ/attribute.go | 10 + internal/command/jsonformat/differ/block.go | 10 + internal/command/jsonformat/differ/output.go | 7 + internal/command/jsonformat/differ/value.go | 103 ++++++++ internal/command/jsonformat/renderer.go | 29 ++ 8 files changed, 497 insertions(+) create mode 100644 internal/command/jsonformat/README.md create mode 100644 internal/command/jsonformat/change/change.go create mode 100644 internal/command/jsonformat/change/renderer.go create mode 100644 internal/command/jsonformat/differ/attribute.go create mode 100644 internal/command/jsonformat/differ/block.go create mode 100644 internal/command/jsonformat/differ/output.go create mode 100644 internal/command/jsonformat/differ/value.go create mode 100644 internal/command/jsonformat/renderer.go diff --git a/internal/command/jsonformat/README.md b/internal/command/jsonformat/README.md new file mode 100644 index 0000000000..3b475799ef --- /dev/null +++ b/internal/command/jsonformat/README.md @@ -0,0 +1,247 @@ +**Note: the implementations discussed here are incomplete, check the chain of +PRs in the various PRs modifying this package to see each new thing being +implemented.** + +# jsonformat + +This package contains functionality around formatting and displaying the JSON +structured output produced by adding the `-json` flag to various Terraform +commands. + +## Terraform Structured Plan Renderer + +As of January 2023, this package contains only a single structure: the +`Renderer`. + +The renderer accepts the JSON structured output produced by the +`terraform show -json` command and writes it in a human-readable +format. + +Implementation details and decisions for the `Renderer` are discussed in the +following sections. + +### Implementation + +There are two subpackages within the `jsonformat` renderer package. The `differ` +package compares the `before` and `after` values of the given plan and produces +`Change` objects from the `change` package. + +This approach is aimed at ensuring the process by which the plan difference is +calculated is separated from the rendering itself. In this way it should be +possible to modify the rendering or add new renderer formats without being +concerned with the complex diff calculations. + +#### The `differ` package + +The `differ` package operates on `Value` objects. These are produced from +`jsonplan.Change` objects (which are produced by the `terraform show` command). +Each `jsonplan.Change` object represents a single resource within the overall +Terraform configuration. + +The differ package will iterate through the `Value` objects and produce a single +`Change` that represents a processed summary of the changes described by the +`Value`. You will see that the produced changes are nested so a change to a list +attribute will contain a slice of changes, this is discussed in the +"[The change package](#the-change-package)" section. + +##### The `Value` object + +The `Value` objects contains raw Golang representations of JSON objects (generic +`interface{}` fields). These are produced by parsing the `json.RawMessage` +objects within the provided changes. + +The fields the differ cares about from the provided changes are: + +- `Before`: The value before the proposed change. +- `After`: The value after the proposed change. +- `Unknown`: If the value is being computed during the change. +- `BeforeSensitive`: If the value was sensitive before the change. +- `AfterSensitive`: If the value is sensitive after the change. +- `ReplacePaths`: If the change is causing the overall resource to be replaced. + +In addition, the values define two additional meta fields that they set and +manipulate internally: + +- `BeforeExplicit`: If the value in `Before` is explicit or an implied result due to a change elsewhere. +- `AfterExplicit`: If the value in `After` is explicit or an implied result due to a change elsewhere. + +The actual concrete type of each of the generic fields is determined by the +overall schema. The values are also recursive, this means as we iterate through +the `Value` we create relevant child values based on the schema for the given +resource. + +For example, the initial change is always a `block` type which means the +`Before` and `After` values will actually be `map[string]interface{}` types +mapping each attribute and block to their relevant values. The +`Unknown`, `BeforeSensitive`, `AfterSensitive` values will all be either a +`map[string]interface{}` which maps each attribute or nested block to their +unknown and sensitive status, or it could simply be a `boolean` which generally +means the entire block and all children are sensitive or computed. + +In total, a `Value` can represent the following types: + +- `Attribute` + - `map`: Values will typically be `map[string]interface{}`. + - `list`: Values will typically be `[]interface{}`. + - `set`: Values will typically be `[]interface{}`. + - `object`: Values will typically be `map[string]interface{}`. + - `tuple`: Values will typically be `[]interface{}`. + - `bool`: Values will typically be a `bool`. + - `number`: Values will typically be a `float64`. + - `string`: Values will typically be a `string`. +- `Block`: Values will typically be `map[string]interface{}`, but they can be + split between nested blocks and attributes. +- `Output` + - Outputs are interesting as we don't have a schema for them, as such they + can be any JSON type. + - We also use the Output type to represent dynamic attributes, since in both + cases we work out the type based on the JSON representation instead of the + schema. + +The `ReplacePaths` field is unique in that it's value doesn't actually change +based on the schema - it's always a slice of index slices. An index in this +context will either be an integer pointing to a child of a set or a list or a +string pointing to the child of a map, object or block. As we iterate through +the value we manipulate the outer slice to remove child slices where the index +doesn't match and propagate paths that do match onto the children. + +*Quick note on explicit vs implicit:* In practice, it is only possible to get +implicit changes when you manipulate a collection. That is to say child values +of a modified collection will insert `nil` entries into the relevant before +or after fields of their child changes to represent their values being deleted +or created. It is also possible for users to explicitly put null values into +their collections, and this behaviour is different to deleting an item in the +collection. With the `BeforeExplicit` and `AfterExplicit` values we can tell the +difference between whether this value was removed from a collection or this +value was set to null in a collection. + +*Quick note on the go-cty Value and Type objects:* The `Before` and `After` +fields are actually go-cty values, but we cannot convert them directly because +of the Terraform Cloud redacted endpoint. The redacted endpoint turns sensitive +values into strings regardless of their types. Because of this, we cannot just +do a direct conversion using the ctyjson package. We would have to iterate +through the schema first, find the sensitive values and their mapped types, +update the types inside the schema to strings, and then go back and do the +overall conversion. This isn't including any of the more complicated parts +around what happens if something was sensitive before and isn't sensitive after +or vice versa. This would mean the type would need to change between the before +and after value. It is in fact just easier to iterate through the values as +generic JSON interfaces, and obfuscate the sensitive values as we never need to +print them anyway. + +##### Iterating through values + +The `differ` package will recursively create child `Value` objects for the +complex objects. + +There are two key subtypes of a `Value`: `SliceValue` and `MapValue`. +`SliceValue` values are used by list, set, and tuple attributes. `MapValue` +values are used by map and object attributes, and blocks. For what it is worth +outputs and dynamic types can end up using both, but they're kind of special as +the processing for dynamic types works out the type from the JSON struct and +then just passes it into the relevant real types for actual processing. + +The two subtypes implement `GetChild` functions that retrieve a child change +for a relevant index (`int` for slice, `string` for map). These functions build +an entirely populated `Value` object, and the package will then recursively +compute the change for the child (and all other children). When a complex change +has all the children changes, it then passes that into the relevant complex +change type. + +#### The `change` package + +A `Change` should contain all the relevant information it needs to render +itself. + +The `Change` itself contains the action (eg. `Create`, `Delete`, `Update`), and +whether this change is causing the overall resource to be replaced (read from +the `ReplacePaths` field discussed in the previous section). The actual content +of the changes is passed directly into the internal renderer field. The internal +renderer is then an implementation that knows the actual content of the changes +and what they represent. + +For example to instantiate a change resulting from updating a list of +primitives: + +```go + listChange := change.New(change.List([]change.Change{ + change.New(change.Primitive(0.0, 0.0, cty.Number), plans.NoOp, false), + change.New(change.Primitive(1.0, nil, cty.Number), plans.Delete, false), + change.New(change.Primitive(nil, 4.0, cty.Number), plans.Create, false), + change.New(change.Primitive(2.0, 2.0, cty.Number), plans.NoOp, false) + }, plans.Update, false)) +``` + +##### The `Render` function + +Currently, there is only one way to render a change, and it is implemented via +the `Render` function. In the future, there may be additional rendering +capabilities, but for now the `Render` function just passes the call directly +onto the internal renderer. + +Rendering the change with: `listChange.Render(0, RenderOpts{})` would +produce: + +```text +[ + 0, + - 1 -> null, + + 4, + 2, +] +``` + +Note, the render function itself doesn't print out metadata about its own change +(eg. there's no `~` symbol in front of the opening bracket). The expectation is +that parent changes control how child changes are rendered (so are responsible) +for deciding on their opening indentation, whether they have a key (as in maps, +objects, and blocks), or how the action symbol is displayed. + +In the above example, the primitive renderer would print out only `1 -> null` +while the surrounding list renderer is providing the indentation, the symbol and +the line ending commas. + +##### Implementing new change types + +To implement a new change type, you must implement the internal Renderer +functionality. To do this you create a new implementation of the +`change/renderer.go`, make sure it accepts all the data you need, and implement +the `Render` function (and any other additional render functions that may +exist). + +Some changes publish warnings that should be displayed alongside them. +If your new change has no warnings you can use the `NoWarningsRenderer` to avoid +implementing the additional `Warnings` function. + +If/when new Renderer types are implemented, additional `Render` like functions +will be added. You should implement all of these with your new change type. + +##### Implementing new renderer types for changes + +As of January 2023, there is only a single type of renderer (the human-readable) +renderer. As such, the `Change` structure provides a single `Render` function. + +To implement a new renderer: + +1. Add a new render function onto the internal `Renderer` interface. +2. Add a new render function onto the Change struct that passes the call onto + the internal renderer. +3. Implement the new function on all the existing internal interfaces. + +Since each internal renderer contains all the information it needs to provide +change information about itself, your new Render function should pass in +anything it needs. + +### New types of Renderer + +In the future, we may wish to add in different kinds of renderer, such as a +compact renderer, or an interactive renderer. To do this, you'll need to modify +the Renderer struct or create a new type of Renderer. + +The logic around creating the `Change` structures will be shared (ie. calling +into the differ package should be consistent across renderers). But when it +comes to rendering the changes, I'd expect the `Change` structures to implement +additional functions that allow them to internally organise the data as required +and return a relevant object. For the existing human-readable renderer that is +simply a string, but for a future interactive renderer it might be a model from +an MVC pattern. diff --git a/internal/command/jsonformat/change/change.go b/internal/command/jsonformat/change/change.go new file mode 100644 index 0000000000..34e085a1d8 --- /dev/null +++ b/internal/command/jsonformat/change/change.go @@ -0,0 +1,57 @@ +package change + +import "github.com/hashicorp/terraform/internal/plans" + +// Change captures a change to a single block, element or attribute. +// +// It essentially merges common functionality across all types of changes, +// namely the replace logic and the action / change type. Any remaining +// behaviour can be offloaded to the renderer which will be unique for the +// various change types (eg. maps, objects, lists, blocks, primitives, etc.). +type Change struct { + // renderer captures the uncommon functionality across the different kinds + // of changes. Each type of change (lists, blocks, sets, etc.) will have a + // unique renderer. + renderer Renderer + + // action is the action described by this change (such as create, delete, + // update, etc.). + action plans.Action + + // replace tells the Change that it should add the `# forces replacement` + // suffix. + // + // Every single change could potentially add this suffix, so we embed it in + // the change as common functionality instead of in the specific renderers. + replace bool +} + +// New creates a new Change object with the provided renderer, action and +// replace context. +func New(renderer Renderer, action plans.Action, replace bool) Change { + return Change{ + renderer: renderer, + action: action, + replace: replace, + } +} + +// Render prints the Change into a human-readable string referencing the +// specified RenderOpts. +// +// If the returned string is a single line, then indent should be ignored. +// +// If the return string is multiple lines, then indent should be used to offset +// the beginning of all lines but the first by the specified amount. +func (change Change) Render(indent int, opts RenderOpts) string { + return change.renderer.Render(change, indent, opts) +} + +// Warnings returns a list of strings that should be rendered as warnings before +// a given change is rendered. +// +// As with the Render function, the indent should only be applied on multiline +// warnings and on the second and following lines. +func (change Change) Warnings(indent int) []string { + return change.renderer.Warnings(change, indent) +} diff --git a/internal/command/jsonformat/change/renderer.go b/internal/command/jsonformat/change/renderer.go new file mode 100644 index 0000000000..98cdd876d7 --- /dev/null +++ b/internal/command/jsonformat/change/renderer.go @@ -0,0 +1,34 @@ +package change + +// Renderer renders a specific change. +// +// Implementations should handle unique functionality relevant to the specific +// change type. Any common functionality shared between multiple change +// renderers should be pushed into the Change structure itself. +type Renderer interface { + Render(change Change, indent int, opts RenderOpts) string + Warnings(change Change, indent int) []string +} + +// NoWarningsRenderer defines a Warnings function that returns an empty list of +// warnings. This can be used by other renderers to ensure we don't see lots of +// repeats of this empty function. +type NoWarningsRenderer struct{} + +// Warnings returns an empty slice, as the name NoWarningsRenderer suggests. +func (render NoWarningsRenderer) Warnings(change Change, indent int) []string { + return nil +} + +// RenderOpts contains options that can control how the Renderer.Render function +// will render. +// +// For now, we haven't implemented any of the Renderer functionality, so we have +// no options currently. +type RenderOpts struct{} + +// Clone returns a new RenderOpts object, that matches the original but can be +// edited without changing the original. +func (opts RenderOpts) Clone() RenderOpts { + return RenderOpts{} +} diff --git a/internal/command/jsonformat/differ/attribute.go b/internal/command/jsonformat/differ/attribute.go new file mode 100644 index 0000000000..869dc760b8 --- /dev/null +++ b/internal/command/jsonformat/differ/attribute.go @@ -0,0 +1,10 @@ +package differ + +import ( + "github.com/hashicorp/terraform/internal/command/jsonformat/change" + "github.com/hashicorp/terraform/internal/command/jsonprovider" +) + +func (v Value) ComputeChangeForAttribute(attribute *jsonprovider.Attribute) change.Change { + panic("not implemented") +} diff --git a/internal/command/jsonformat/differ/block.go b/internal/command/jsonformat/differ/block.go new file mode 100644 index 0000000000..3b0dea8461 --- /dev/null +++ b/internal/command/jsonformat/differ/block.go @@ -0,0 +1,10 @@ +package differ + +import ( + "github.com/hashicorp/terraform/internal/command/jsonformat/change" + "github.com/hashicorp/terraform/internal/command/jsonprovider" +) + +func (v Value) ComputeChangeForBlock(block *jsonprovider.Block) change.Change { + panic("not implemented") +} diff --git a/internal/command/jsonformat/differ/output.go b/internal/command/jsonformat/differ/output.go new file mode 100644 index 0000000000..fe85bb1053 --- /dev/null +++ b/internal/command/jsonformat/differ/output.go @@ -0,0 +1,7 @@ +package differ + +import "github.com/hashicorp/terraform/internal/command/jsonformat/change" + +func (v Value) ComputeChangeForOutput() change.Change { + panic("not implemented") +} diff --git a/internal/command/jsonformat/differ/value.go b/internal/command/jsonformat/differ/value.go new file mode 100644 index 0000000000..bd927be723 --- /dev/null +++ b/internal/command/jsonformat/differ/value.go @@ -0,0 +1,103 @@ +package differ + +import ( + "encoding/json" + + "github.com/hashicorp/terraform/internal/command/jsonplan" +) + +// Value contains the unmarshalled generic interface{} types that are output by +// the JSON functions in the various json packages (such as jsonplan and +// jsonprovider). +// +// A Value can be converted into a change.Change, ready for rendering, with the +// ComputeChangeForAttribute, ComputeChangeForOutput, and ComputeChangeForBlock +// functions. +// +// The Before and After fields are actually go-cty values, but we cannot convert +// them directly because of the Terraform Cloud redacted endpoint. The redacted +// endpoint turns sensitive values into strings regardless of their types. +// Because of this, we cannot just do a direct conversion using the ctyjson +// package. We would have to iterate through the schema first, find the +// sensitive values and their mapped types, update the types inside the schema +// to strings, and then go back and do the overall conversion. This isn't +// including any of the more complicated parts around what happens if something +// was sensitive before and isn't sensitive after or vice versa. This would mean +// the type would need to change between the before and after value. It is in +// fact just easier to iterate through the values as generic JSON interfaces. +type Value struct { + // BeforeExplicit refers to whether the Before value is explicit or + // implicit. It is explicit if it has been specified by the user, and + // implicit if it has been set as a consequence of other changes. + // + // For example, explicitly setting a value to null in a list should result + // in Before being null and BeforeExplicit being true. In comparison, + // removing an element from a list should also result in Before being null + // and BeforeExplicit being false. Without the explicit information our + // functions would not be able to tell the difference between these two + // cases. + BeforeExplicit bool + + // AfterExplicit matches BeforeExplicit except references the After value. + AfterExplicit bool + + // Before contains the value before the proposed change. + // + // The type of the value should be informed by the schema and cast + // appropriately when needed. + Before interface{} + + // After contains the value after the proposed change. + // + // The type of the value should be informed by the schema and cast + // appropriately when needed. + After interface{} + + // Unknown describes whether the After value is known or unknown at the time + // of the plan. In practice, this means the after value should be rendered + // simply as `(known after apply)`. + // + // The concrete value could be a boolean describing whether the entirety of + // the After value is unknown, or it could be a list or a map depending on + // the schema describing whether specific elements or attributes within the + // value are unknown. + Unknown interface{} + + // BeforeSensitive matches Unknown, but references whether the Before value + // is sensitive. + BeforeSensitive interface{} + + // AfterSensitive matches Unknown, but references whether the After value is + // sensitive. + AfterSensitive interface{} + + // ReplacePaths generally contains nested slices that describe paths to + // elements or attributes that are causing the overall resource to be + // replaced. + ReplacePaths interface{} +} + +// ValueFromJsonChange unmarshals the raw []byte values in the jsonplan.Change +// structs into generic interface{} types that can be reasoned about. +func ValueFromJsonChange(change jsonplan.Change) Value { + return Value{ + Before: unmarshalGeneric(change.Before), + After: unmarshalGeneric(change.After), + Unknown: unmarshalGeneric(change.AfterUnknown), + BeforeSensitive: unmarshalGeneric(change.BeforeSensitive), + AfterSensitive: unmarshalGeneric(change.AfterSensitive), + ReplacePaths: unmarshalGeneric(change.ReplacePaths), + } +} + +func unmarshalGeneric(raw json.RawMessage) interface{} { + if raw == nil { + return nil + } + + var out interface{} + if err := json.Unmarshal(raw, &out); err != nil { + panic("unrecognized json type: " + err.Error()) + } + return out +} diff --git a/internal/command/jsonformat/renderer.go b/internal/command/jsonformat/renderer.go new file mode 100644 index 0000000000..680c56ef2d --- /dev/null +++ b/internal/command/jsonformat/renderer.go @@ -0,0 +1,29 @@ +package jsonformat + +import ( + "github.com/mitchellh/colorstring" + + "github.com/hashicorp/terraform/internal/command/jsonplan" + "github.com/hashicorp/terraform/internal/command/jsonprovider" + "github.com/hashicorp/terraform/internal/terminal" +) + +type Plan struct { + OutputChanges map[string]jsonplan.Change `json:"output_changes"` + ResourceChanges []jsonplan.ResourceChange `json:"resource_changes"` + ResourceDrift []jsonplan.ResourceChange `json:"resource_drift"` + ProviderSchemas map[string]*jsonprovider.Provider `json:"provider_schemas"` +} + +type Renderer struct { + Streams *terminal.Streams + Colorize *colorstring.Colorize +} + +func (r Renderer) RenderPlan(plan Plan) { + panic("not implemented") +} + +func (r Renderer) RenderLog(message map[string]interface{}) { + panic("not implemented") +}