terraform console: Multi-line entry support

The console command, when running in interactive mode, will now detect if
the input seems to be an incomplete (but valid enough so far) expression,
and if so will produce another prompt to accept another line of expression
input.

This is primarily to make it easier to paste in multi-line expressions
taken from elsewhere, but it could also be used for manual input.
The support for multi-line _editing_ is limited by the fact that the
readline dependency we use doesn't support multiline input and so we're
currently doing this in spite of that library. Hopefully we'll be able to
improve on that in future either by contributing multi-line editing support
upstream or by switching to a different readline dependency.

The delimiter-counting heuristic employed here is similar to the one used
by HCL itself to decide whether a newline should end the definition of an
attribute, but this implementation is simpler because it doesn't need to
produce error messages or perform any recovery. Instead, it just bails out
if it encounters something strange so that the console session can return
a parse error.

Because some invalid input may cause a user to become "stuck" in a multi-
line sequence, we consider a blank line as intent to immediately try to
evaluate what was entered, and also interpret SIGINT (e.g. Ctrl+C) as
cancellation of multi-line input, assuming that at least one line was
already entered, extending the previous precedent that SIGINT cancels
when at least one character was already entered at the prompt.

internal/command/console_interactive.go

@@ -13,11 +13,12 @@ import (
 	"fmt"
 	"io"
 	"os"
-
-	"github.com/hashicorp/terraform/internal/repl"
+	"strings"
 
 	"github.com/chzyer/readline"
 	"github.com/hashicorp/cli"
+
+	"github.com/hashicorp/terraform/internal/repl"
 )
 
 func (c *ConsoleCommand) modeInteractive(session *repl.Session, ui cli.Ui) int {
@@ -39,20 +40,55 @@ func (c *ConsoleCommand) modeInteractive(session *repl.Session, ui cli.Ui) int {
 	}
 	defer l.Close()
 
+	// TODO: Currently we're handling multi-line input largely _in spite of_
+	// the readline library, because it doesn't support that. This means that
+	// in particular the history treats each line as a separate history entry,
+	// and doesn't allow editing of previous lines after the user's already
+	// pressed enter.
+	//
+	// Hopefully we can do better than this one day, but having some basic
+	// support for multi-line input is at least better than none at all:
+	// this is mainly helpful when pasting in expressions from elsewhere that
+	// already have newline characters in them, to avoid pre-editing it.
+
+	lines := make([]string, 0, 4)
 	for {
 		// Read a line
+		if len(lines) == 0 {
+			l.SetPrompt("> ")
+		} else {
+			l.SetPrompt(": ")
+		}
 		line, err := l.Readline()
 		if err == readline.ErrInterrupt {
-			if len(line) == 0 {
+			if len(lines) == 0 && line == "" {
 				break
+			} else if line != "" {
+				continue
 			} else {
+				// Reset the entry buffer to start a new expression
+				lines = lines[:0]
+				ui.Output("(multi-line entry canceled)")
 				continue
 			}
 		} else if err == io.EOF {
 			break
 		}
+		lines = append(lines, line)
+		// The following implements a heuristic for deciding if it seems likely
+		// that the user was intending to continue entering more expression
+		// characters on a subsequent line. This should get the right answer
+		// for any valid expression, but might get confused by invalid input.
+		// The user can always hit enter one more time (entering a blank line)
+		// to break out of a multi-line sequence and force interpretation of
+		// what was already entered.
+		if repl.ExpressionEntryCouldContinue(lines) {
+			continue
+		}
 
-		out, exit, diags := session.Handle(line)
+		input := strings.Join(lines, "\n") + "\n"
+		lines = lines[:0] // reset for next iteration
+		out, exit, diags := session.Handle(input)
 		if diags.HasErrors() {
 			c.showDiagnostics(diags)
 		}

internal/repl/continuation.go

@@ -0,0 +1,96 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package repl
+
+import (
+	"strings"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+)
+
+// ExpressionEntryCouldContinue is a helper for terraform console's interactive
+// mode which serves as a heuristic for whether it seems like the author might
+// be trying to split an expression over multiple lines of input.
+//
+// The current heuristic is whether there's at least one bracketing delimiter
+// that isn't closed, but only if any closing brackets already present are
+// properly balanced.
+//
+// This function also always returns false if the last line entered is empty,
+// because that seems likely to represent a user trying to force Terraform to
+// accept something that didn't pass the heuristic for some reason, at which
+// point Terraform can try to evaluate the expression and return an error if
+// it's invalid syntax.
+func ExpressionEntryCouldContinue(linesSoFar []string) bool {
+	if len(linesSoFar) == 0 || strings.TrimSpace(linesSoFar[len(linesSoFar)-1]) == "" {
+		// If there's no input at all or if the last line is empty other than
+		// spaces, we assume the user is trying to force Terraform to evaluate
+		// what they entered so far without any further continuation.
+		return false
+	}
+
+	// We use capacity 8 here as a compromise assuming that most reasonable
+	// input entered at the console prompt will not use more than eight
+	// levels of nesting, but even if it does then we'll just reallocate the
+	// slice and so it's not a big deal.
+	delimStack := make([]hclsyntax.TokenType, 0, 8)
+	push := func(typ hclsyntax.TokenType) {
+		delimStack = append(delimStack, typ)
+	}
+	pop := func() hclsyntax.TokenType {
+		if len(delimStack) == 0 {
+			return hclsyntax.TokenInvalid
+		}
+		ret := delimStack[len(delimStack)-1]
+		delimStack = delimStack[:len(delimStack)-1]
+		return ret
+	}
+	// We need to scan this all as one string because the HCL lexer has a few
+	// special cases where it tracks open/close state itself, such as in heredocs.
+	all := strings.Join(linesSoFar, "\n") + "\n"
+	toks, diags := hclsyntax.LexExpression([]byte(all), "", hcl.InitialPos)
+	if diags.HasErrors() {
+		return false // bail early if the input is already invalid
+	}
+	for _, tok := range toks {
+		switch tok.Type {
+		case hclsyntax.TokenOBrace, hclsyntax.TokenOBrack, hclsyntax.TokenOParen, hclsyntax.TokenOHeredoc, hclsyntax.TokenTemplateInterp, hclsyntax.TokenTemplateControl:
+			// Opening delimiters go on our stack so that we can hopefully
+			// match them with closing delimiters later.
+			push(tok.Type)
+		case hclsyntax.TokenCBrace:
+			open := pop()
+			if open != hclsyntax.TokenOBrace {
+				return false
+			}
+		case hclsyntax.TokenCBrack:
+			open := pop()
+			if open != hclsyntax.TokenOBrack {
+				return false
+			}
+		case hclsyntax.TokenCParen:
+			open := pop()
+			if open != hclsyntax.TokenOParen {
+				return false
+			}
+		case hclsyntax.TokenCHeredoc:
+			open := pop()
+			if open != hclsyntax.TokenOHeredoc {
+				return false
+			}
+		case hclsyntax.TokenTemplateSeqEnd:
+			open := pop()
+			if open != hclsyntax.TokenTemplateInterp && open != hclsyntax.TokenTemplateControl {
+				return false
+			}
+		}
+	}
+
+	// If we get here without returning early then all of the closing delimeters
+	// were matched by opening delimiters. If our stack still contains at least
+	// one opening bracket then it seems like the user is intending to type
+	// more.
+	return len(delimStack) != 0
+}

internal/repl/continuation_test.go

@@ -0,0 +1,307 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package repl
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestExpressionEntryCouldContinue(t *testing.T) {
+	tests := []struct {
+		Input []string
+		Want  bool
+	}{
+		{
+			nil,
+			false,
+		},
+		{
+			[]string{
+				"",
+			},
+			false,
+		},
+		{
+			[]string{
+				"foo(",
+				"", // trailing newline forces termination
+			},
+			false,
+		},
+
+		// parens
+		{
+			[]string{
+				"foo()",
+			},
+			false,
+		},
+		{
+			[]string{
+				"foo(",
+			},
+			true,
+		},
+		{
+			[]string{
+				"foo(",
+				"  bar,",
+				")",
+			},
+			false,
+		},
+
+		// brackets
+		{
+			[]string{
+				"[]",
+			},
+			false,
+		},
+		{
+			[]string{
+				"[",
+			},
+			true,
+		},
+		{
+			[]string{
+				"[",
+				"]",
+			},
+			false,
+		},
+
+		// braces
+		{
+			[]string{
+				"{}",
+			},
+			false,
+		},
+		{
+			[]string{
+				"{",
+			},
+			true,
+		},
+		{
+			[]string{
+				"{",
+				"}",
+			},
+			false,
+		},
+
+		// quotes
+		// HCL doesn't allow splitting quoted strings over multiple lines, so
+		// these never cause continuation. (Use heredocs instead for that)
+		{
+			[]string{
+				`""`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"`,
+				`"`,
+			},
+			false,
+		},
+
+		// heredoc templates
+		{
+			[]string{
+				`<<EOT`,
+				`EOT`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`<<EOT`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`<<EOT`,
+				`beep`,
+				`EOT`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`<<EOT`,
+				`beep`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`<<-EOT`,
+				`EOT`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`<<-EOT`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`<<-EOT`,
+				`beep`,
+				`EOT`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`<<-EOT`,
+				`beep`,
+			},
+			true,
+		},
+		{
+			// In the following it's actually the heredoc that's keeping the
+			// newline sequence going, rather than the control sequence, but
+			// this is here to test a reasonable combination of things someone
+			// might enter.
+			[]string{
+				`<<EOT`,
+				`%{ for x in y }`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`<<EOT`,
+				`%{ for x in y }`,
+				`boop`,
+				`%{ endfor }`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`<<EOT`,
+				`%{ for x in y }`,
+				`boop`,
+				`%{ endfor }`,
+				`EOT`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`<<EOT`,
+				`]`, // literal bracket, so doesn't count as a mismatch
+			},
+			true,
+		},
+
+		// template interpolation/control inside quotes
+		// although quotes alone cannot span over multiple lines, a
+		// template sequence creates a nested context where newlines are
+		// allowed.
+		{
+			[]string{
+				`"${hello}"`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"${hello`,
+				`}"`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"${`,
+				`  hello`,
+				`}"`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"${`,
+				`  hello`,
+			},
+			true,
+		},
+		{
+			[]string{
+				`"%{ for x in y }%{ endfor }"`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"%{`,
+				`   for x in y }%{ endfor }"`,
+			},
+			false,
+		},
+		{
+			// This case returns false because the control sequence itself
+			// ends before the newline, and quoted literals are not allowed
+			// to contain newlines, so this is a parse error.
+			[]string{
+				`"%{ for x in y }`,
+			},
+			false,
+		},
+
+		// mismatched brackets
+		// these combinations should always return false so that we can
+		// report the syntax error immediately.
+		{
+			[]string{
+				`([)`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"${]`,
+			},
+			false,
+		},
+		{
+			[]string{
+				`"%{]`,
+			},
+			false,
+		},
+	}
+
+	for _, test := range tests {
+		name := strings.Join(test.Input, "⮠")
+		t.Run(name, func(t *testing.T) {
+			got := ExpressionEntryCouldContinue(test.Input)
+			if got != test.Want {
+				t.Errorf(
+					"wrong result\ninput:\n%s\ngot:  %t\nwant: %t",
+					strings.Join(test.Input, "\n"),
+					got, test.Want,
+				)
+			}
+		})
+	}
+}