add policy evaluation task stage

internal/cloud/backend_taskStage_policyEvaluation.go

@@ -0,0 +1,157 @@
+package cloud
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-tfe"
+)
+
+type policyEvaluationSummary struct {
+	unreachable bool
+	pending     int
+	failed      int
+	passed      int
+}
+
+type Symbol rune
+
+const (
+	Tick          Symbol = '\u2713'
+	Cross         Symbol = '\u00d7'
+	Warning       Symbol = '\u24be'
+	Arrow         Symbol = '\u2192'
+	DownwardArrow Symbol = '\u21b3'
+)
+
+type policyEvaluationSummarizer struct {
+	finished bool
+	cloud    *Cloud
+	counter  int
+}
+
+func newPolicyEvaluationSummarizer(b *Cloud, ts *tfe.TaskStage) taskStageSummarizer {
+	if len(ts.PolicyEvaluations) == 0 {
+		return nil
+	}
+	return &policyEvaluationSummarizer{
+		finished: false,
+		cloud:    b,
+	}
+}
+
+func (pes *policyEvaluationSummarizer) Summarize(context *IntegrationContext, output IntegrationOutputWriter, ts *tfe.TaskStage) (bool, *string, error) {
+	if pes.counter == 0 {
+		output.Output("[bold]OPA Policy Evaluation\n")
+		pes.counter++
+	}
+
+	if pes.finished {
+		return false, nil, nil
+	}
+
+	counts := summarizePolicyEvaluationResults(ts.PolicyEvaluations)
+
+	if counts.pending != 0 {
+		pendingMessage := "Evaluating ... "
+		return true, &pendingMessage, nil
+	}
+
+	if counts.unreachable {
+		output.Output("Skipping policy evaluation.")
+		output.End()
+		return false, nil, nil
+	}
+
+	// Print out the summary
+	if err := pes.taskStageWithPolicyEvaluation(context, output, ts.PolicyEvaluations); err != nil {
+		return false, nil, err
+	}
+	// Mark as finished
+	pes.finished = true
+
+	return false, nil, nil
+}
+
+func summarizePolicyEvaluationResults(policyEvaluations []*tfe.PolicyEvaluation) *policyEvaluationSummary {
+	var pendingCount, errCount, passedCount int
+	for _, policyEvaluation := range policyEvaluations {
+		switch policyEvaluation.Status {
+		case "unreachable":
+			return &policyEvaluationSummary{
+				unreachable: true,
+			}
+		case "running", "pending", "queued":
+			pendingCount++
+		case "passed":
+			passedCount++
+		default:
+			// Everything else is a failure
+			errCount++
+		}
+	}
+
+	return &policyEvaluationSummary{
+		unreachable: false,
+		pending:     pendingCount,
+		failed:      errCount,
+		passed:      passedCount,
+	}
+}
+
+func (pes *policyEvaluationSummarizer) taskStageWithPolicyEvaluation(context *IntegrationContext, output IntegrationOutputWriter, policyEvaluation []*tfe.PolicyEvaluation) error {
+	var result, message string
+	// Currently only one policy evaluation supported : OPA
+	for _, polEvaluation := range policyEvaluation {
+		if polEvaluation.Status == tfe.PolicyEvaluationPassed {
+			message = "[dim] This result means that all OPA policies passed and the protected behaviour is allowed"
+			result = fmt.Sprintf("[green]%s", strings.ToUpper(string(tfe.PolicyEvaluationPassed)))
+			if polEvaluation.ResultCount.AdvisoryFailed > 0 {
+				result += " (with advisory)"
+			}
+		} else {
+			message = "[dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules"
+			result = fmt.Sprintf("[red]%s", strings.ToUpper(string(tfe.PolicyEvaluationFailed)))
+		}
+
+		output.Output(fmt.Sprintf("[bold]%c%c Overall Result: %s", Arrow, Arrow, result))
+
+		output.Output(message)
+
+		total := getPolicyCount(polEvaluation.ResultCount)
+
+		output.Output(fmt.Sprintf("%d policies evaluated\n", total))
+
+		policyOutcomes, err := pes.cloud.client.PolicySetOutcomes.List(context.StopContext, polEvaluation.ID, nil)
+		if err != nil {
+			return err
+		}
+
+		for i, out := range policyOutcomes.Items {
+			output.Output(fmt.Sprintf("%c Policy set %d: [bold]%s (%d)", Arrow, i+1, out.PolicySetName, len(out.Outcomes)))
+			for _, outcome := range out.Outcomes {
+				output.Output(fmt.Sprintf("  %c Policy name: [bold]%s", DownwardArrow, outcome.PolicyName))
+				switch outcome.Status {
+				case "passed":
+					output.Output(fmt.Sprintf("     | [green][bold]%c Passed", Tick))
+				case "failed":
+					if outcome.EnforcementLevel == tfe.EnforcementAdvisory {
+						output.Output(fmt.Sprintf("     | [blue][bold]%c Advisory", Warning))
+					} else {
+						output.Output(fmt.Sprintf("     | [red][bold]%c Failed", Cross))
+					}
+				}
+				if outcome.Description != "" {
+					output.Output(fmt.Sprintf("     | [dim]%s", outcome.Description))
+				} else {
+					output.Output("     | [dim]No description available")
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func getPolicyCount(resultCount *tfe.PolicyResultCount) int {
+	return resultCount.AdvisoryFailed + resultCount.MandatoryFailed + resultCount.Errored + resultCount.Passed
+}

internal/cloud/backend_taskStage_policyEvaluation_test.go

@@ -0,0 +1,97 @@
+package cloud
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/go-tfe"
+)
+
+func TestCloud_runTaskStageWithPolicyEvaluation(t *testing.T) {
+	b, bCleanup := testBackendWithName(t)
+	defer bCleanup()
+
+	integrationContext, writer := newMockIntegrationContext(b, t)
+
+	cases := map[string]struct {
+		taskStage       func() *tfe.TaskStage
+		context         *IntegrationContext
+		writer          *testIntegrationOutput
+		expectedOutputs []string
+		isError         bool
+	}{
+		"all-succeeded": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "pol-pass", ResultCount: &tfe.PolicyResultCount{Passed: 1}, Status: "passed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [green]PASSED\n│ [dim] This result means that all OPA policies passed and the protected behaviour is allowed\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-passes (1)\n│   ↳ Policy name: [bold]policy-pass\n│      | [green][bold]✓ Passed\n│      | [dim]This policy will pass\n"},
+			isError:         false,
+		},
+		"mandatory-failed": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "pol-fail", ResultCount: &tfe.PolicyResultCount{MandatoryFailed: 1}, Status: "failed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│   ↳ Policy name: [bold]policy-fail\n│      | [red][bold]× Failed\n│      | [dim]This policy will fail"},
+			isError:         true,
+		},
+		"advisory-failed": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{AdvisoryFailed: 1}, Status: "failed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│   ↳ Policy name: [bold]policy-fail\n│      | [blue][bold]Ⓘ Advisory\n│      | [dim]This policy will fail"},
+			isError:         false,
+		},
+		"unreachable": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{Errored: 1}, Status: "unreachable"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"Skipping policy evaluation."},
+			isError:         false,
+		},
+	}
+
+	for _, c := range cases {
+		c.writer.output.Reset()
+		trs := policyEvaluationSummarizer{
+			cloud: b,
+		}
+		c.context.Poll(func(i int) (bool, error) {
+			cont, _, _ := trs.Summarize(c.context, c.writer, c.taskStage())
+			if cont {
+				return true, nil
+			}
+
+			output := c.writer.output.String()
+			for _, expected := range c.expectedOutputs {
+				if !strings.Contains(output, expected) {
+					t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output)
+				}
+			}
+			return false, nil
+		})
+	}
+}

internal/cloud/backend_taskStages.go

@@ -44,7 +44,7 @@ func (b *Cloud) runTaskStages(ctx context.Context, client *tfe.Client, runId str
 
 func (b *Cloud) getTaskStageWithAllOptions(ctx *IntegrationContext, stageID string) (*tfe.TaskStage, error) {
 	options := tfe.TaskStageReadOptions{
-		Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults},
+		Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
 	}
 	stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
 	if err != nil {
@@ -63,13 +63,18 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr
 	if err != nil {
 		return err
 	}
+
 	if s := newTaskResultSummarizer(b, ts); s != nil {
 		summarizers = append(summarizers, s)
 	}
 
+	if s := newPolicyEvaluationSummarizer(b, ts); s != nil {
+		summarizers = append(summarizers, s)
+	}
+
 	return ctx.Poll(func(i int) (bool, error) {
 		options := tfe.TaskStageReadOptions{
-			Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults},
+			Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
 		}
 		stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
 		if err != nil {
@@ -98,7 +103,25 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr
 					errs.Append(err)
 				}
 			}
-		case "unreachable":
+		case tfe.TaskStageAwaitingOverride:
+			// TODO: Add override functionality
+			for _, s := range summarizers {
+				cont, msg, err := s.Summarize(ctx, output, stage)
+				if cont {
+					if msg != nil {
+						if i%4 == 0 {
+							if i > 0 {
+								output.OutputElapsed(*msg, len(*msg)) // Up to 2 digits are allowed by the max message allocation
+							}
+						}
+					}
+					return true, nil
+				}
+				if err != nil {
+					errs.Append(err)
+				}
+			}
+		case tfe.TaskStageUnreachable:
 			return false, nil
 		default:
 			return false, fmt.Errorf("Invalid Task stage status: %s ", stage.Status)

internal/cloud/testing.go

@@ -209,6 +209,7 @@ func testBackend(t *testing.T, obj cty.Value) (*Cloud, func()) {
 	b.client.CostEstimates = mc.CostEstimates
 	b.client.Organizations = mc.Organizations
 	b.client.Plans = mc.Plans
+	b.client.PolicySetOutcomes = mc.PolicySetOutcomes
 	b.client.PolicyChecks = mc.PolicyChecks
 	b.client.Runs = mc.Runs
 	b.client.StateVersions = mc.StateVersions
@@ -269,6 +270,7 @@ func testUnconfiguredBackend(t *testing.T) (*Cloud, func()) {
 	b.client.CostEstimates = mc.CostEstimates
 	b.client.Organizations = mc.Organizations
 	b.client.Plans = mc.Plans
+	b.client.PolicySetOutcomes = mc.PolicySetOutcomes
 	b.client.PolicyChecks = mc.PolicyChecks
 	b.client.Runs = mc.Runs
 	b.client.StateVersions = mc.StateVersions

internal/cloud/tfe_client_mock.go

@@ -27,6 +27,7 @@ type MockClient struct {
 	CostEstimates         *MockCostEstimates
 	Organizations         *MockOrganizations
 	Plans                 *MockPlans
+	PolicySetOutcomes     *MockPolicySetOutcomes
 	PolicyChecks          *MockPolicyChecks
 	Runs                  *MockRuns
 	StateVersions         *MockStateVersions
@@ -42,6 +43,7 @@ func NewMockClient() *MockClient {
 	c.CostEstimates = newMockCostEstimates(c)
 	c.Organizations = newMockOrganizations(c)
 	c.Plans = newMockPlans(c)
+	c.PolicySetOutcomes = newMockPolicySetOutcomes(c)
 	c.PolicyChecks = newMockPolicyChecks(c)
 	c.Runs = newMockRuns(c)
 	c.StateVersions = newMockStateVersions(c)
@@ -545,6 +547,134 @@ func (m *MockPlans) ReadJSONOutput(ctx context.Context, planID string) ([]byte,
 	return []byte(planOutput), nil
 }
 
+type MockPolicySetOutcomes struct {
+	client *MockClient
+}
+
+func newMockPolicySetOutcomes(client *MockClient) *MockPolicySetOutcomes {
+	return &MockPolicySetOutcomes{
+		client: client,
+	}
+}
+
+func (m *MockPolicySetOutcomes) List(ctx context.Context, policyEvaluationID string, options *tfe.PolicySetOutcomeListOptions) (*tfe.PolicySetOutcomeList, error) {
+	switch policyEvaluationID {
+	case "pol-pass":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "passed",
+							PolicyName:       "policy-pass",
+							Description:      "This policy will pass",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-passes",
+					PolicySetDescription: "This policy set will always pass",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 0,
+						Passed:          1,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	case "pol-fail":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "failed",
+							PolicyName:       "policy-fail",
+							Description:      "This policy will fail",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-fails",
+					PolicySetDescription: "This policy set will always fail",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 1,
+						Passed:          0,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+
+	case "adv-fail":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "advisory",
+							Query:            "data.example.rule",
+							Status:           "failed",
+							PolicyName:       "policy-fail",
+							Description:      "This policy will fail",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-fails",
+					PolicySetDescription: "This policy set will always fail",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  1,
+						MandatoryFailed: 0,
+						Passed:          0,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	default:
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "passed",
+							PolicyName:       "policy-pass",
+							Description:      "This policy will pass",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-passes",
+					PolicySetDescription: "This policy set will always pass",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 0,
+						Passed:          1,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	}
+}
+
+func (m *MockPolicySetOutcomes) Read(ctx context.Context, policySetOutcomeID string) (*tfe.PolicySetOutcome, error) {
+	return nil, nil
+}
+
 type MockPolicyChecks struct {
 	client *MockClient
 	checks map[string]*tfe.PolicyCheck