diff --git a/internal/cloud/backend_taskStage_policyEvaluation.go b/internal/cloud/backend_taskStage_policyEvaluation.go new file mode 100644 index 0000000000..eeba9591a8 --- /dev/null +++ b/internal/cloud/backend_taskStage_policyEvaluation.go @@ -0,0 +1,157 @@ +package cloud + +import ( + "fmt" + "strings" + + "github.com/hashicorp/go-tfe" +) + +type policyEvaluationSummary struct { + unreachable bool + pending int + failed int + passed int +} + +type Symbol rune + +const ( + Tick Symbol = '\u2713' + Cross Symbol = '\u00d7' + Warning Symbol = '\u24be' + Arrow Symbol = '\u2192' + DownwardArrow Symbol = '\u21b3' +) + +type policyEvaluationSummarizer struct { + finished bool + cloud *Cloud + counter int +} + +func newPolicyEvaluationSummarizer(b *Cloud, ts *tfe.TaskStage) taskStageSummarizer { + if len(ts.PolicyEvaluations) == 0 { + return nil + } + return &policyEvaluationSummarizer{ + finished: false, + cloud: b, + } +} + +func (pes *policyEvaluationSummarizer) Summarize(context *IntegrationContext, output IntegrationOutputWriter, ts *tfe.TaskStage) (bool, *string, error) { + if pes.counter == 0 { + output.Output("[bold]OPA Policy Evaluation\n") + pes.counter++ + } + + if pes.finished { + return false, nil, nil + } + + counts := summarizePolicyEvaluationResults(ts.PolicyEvaluations) + + if counts.pending != 0 { + pendingMessage := "Evaluating ... " + return true, &pendingMessage, nil + } + + if counts.unreachable { + output.Output("Skipping policy evaluation.") + output.End() + return false, nil, nil + } + + // Print out the summary + if err := pes.taskStageWithPolicyEvaluation(context, output, ts.PolicyEvaluations); err != nil { + return false, nil, err + } + // Mark as finished + pes.finished = true + + return false, nil, nil +} + +func summarizePolicyEvaluationResults(policyEvaluations []*tfe.PolicyEvaluation) *policyEvaluationSummary { + var pendingCount, errCount, passedCount int + for _, policyEvaluation := range policyEvaluations { + switch policyEvaluation.Status { + case "unreachable": + return &policyEvaluationSummary{ + unreachable: true, + } + case "running", "pending", "queued": + pendingCount++ + case "passed": + passedCount++ + default: + // Everything else is a failure + errCount++ + } + } + + return &policyEvaluationSummary{ + unreachable: false, + pending: pendingCount, + failed: errCount, + passed: passedCount, + } +} + +func (pes *policyEvaluationSummarizer) taskStageWithPolicyEvaluation(context *IntegrationContext, output IntegrationOutputWriter, policyEvaluation []*tfe.PolicyEvaluation) error { + var result, message string + // Currently only one policy evaluation supported : OPA + for _, polEvaluation := range policyEvaluation { + if polEvaluation.Status == tfe.PolicyEvaluationPassed { + message = "[dim] This result means that all OPA policies passed and the protected behaviour is allowed" + result = fmt.Sprintf("[green]%s", strings.ToUpper(string(tfe.PolicyEvaluationPassed))) + if polEvaluation.ResultCount.AdvisoryFailed > 0 { + result += " (with advisory)" + } + } else { + message = "[dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules" + result = fmt.Sprintf("[red]%s", strings.ToUpper(string(tfe.PolicyEvaluationFailed))) + } + + output.Output(fmt.Sprintf("[bold]%c%c Overall Result: %s", Arrow, Arrow, result)) + + output.Output(message) + + total := getPolicyCount(polEvaluation.ResultCount) + + output.Output(fmt.Sprintf("%d policies evaluated\n", total)) + + policyOutcomes, err := pes.cloud.client.PolicySetOutcomes.List(context.StopContext, polEvaluation.ID, nil) + if err != nil { + return err + } + + for i, out := range policyOutcomes.Items { + output.Output(fmt.Sprintf("%c Policy set %d: [bold]%s (%d)", Arrow, i+1, out.PolicySetName, len(out.Outcomes))) + for _, outcome := range out.Outcomes { + output.Output(fmt.Sprintf(" %c Policy name: [bold]%s", DownwardArrow, outcome.PolicyName)) + switch outcome.Status { + case "passed": + output.Output(fmt.Sprintf(" | [green][bold]%c Passed", Tick)) + case "failed": + if outcome.EnforcementLevel == tfe.EnforcementAdvisory { + output.Output(fmt.Sprintf(" | [blue][bold]%c Advisory", Warning)) + } else { + output.Output(fmt.Sprintf(" | [red][bold]%c Failed", Cross)) + } + } + if outcome.Description != "" { + output.Output(fmt.Sprintf(" | [dim]%s", outcome.Description)) + } else { + output.Output(" | [dim]No description available") + } + } + } + } + return nil +} + +func getPolicyCount(resultCount *tfe.PolicyResultCount) int { + return resultCount.AdvisoryFailed + resultCount.MandatoryFailed + resultCount.Errored + resultCount.Passed +} diff --git a/internal/cloud/backend_taskStage_policyEvaluation_test.go b/internal/cloud/backend_taskStage_policyEvaluation_test.go new file mode 100644 index 0000000000..3d08f74d90 --- /dev/null +++ b/internal/cloud/backend_taskStage_policyEvaluation_test.go @@ -0,0 +1,97 @@ +package cloud + +import ( + "strings" + "testing" + + "github.com/hashicorp/go-tfe" +) + +func TestCloud_runTaskStageWithPolicyEvaluation(t *testing.T) { + b, bCleanup := testBackendWithName(t) + defer bCleanup() + + integrationContext, writer := newMockIntegrationContext(b, t) + + cases := map[string]struct { + taskStage func() *tfe.TaskStage + context *IntegrationContext + writer *testIntegrationOutput + expectedOutputs []string + isError bool + }{ + "all-succeeded": { + taskStage: func() *tfe.TaskStage { + ts := &tfe.TaskStage{} + ts.PolicyEvaluations = []*tfe.PolicyEvaluation{ + {ID: "pol-pass", ResultCount: &tfe.PolicyResultCount{Passed: 1}, Status: "passed"}, + } + return ts + }, + writer: writer, + context: integrationContext, + expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [green]PASSED\n│ [dim] This result means that all OPA policies passed and the protected behaviour is allowed\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-passes (1)\n│ ↳ Policy name: [bold]policy-pass\n│ | [green][bold]✓ Passed\n│ | [dim]This policy will pass\n"}, + isError: false, + }, + "mandatory-failed": { + taskStage: func() *tfe.TaskStage { + ts := &tfe.TaskStage{} + ts.PolicyEvaluations = []*tfe.PolicyEvaluation{ + {ID: "pol-fail", ResultCount: &tfe.PolicyResultCount{MandatoryFailed: 1}, Status: "failed"}, + } + return ts + }, + writer: writer, + context: integrationContext, + expectedOutputs: []string{"│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│ ↳ Policy name: [bold]policy-fail\n│ | [red][bold]× Failed\n│ | [dim]This policy will fail"}, + isError: true, + }, + "advisory-failed": { + taskStage: func() *tfe.TaskStage { + ts := &tfe.TaskStage{} + ts.PolicyEvaluations = []*tfe.PolicyEvaluation{ + {ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{AdvisoryFailed: 1}, Status: "failed"}, + } + return ts + }, + writer: writer, + context: integrationContext, + expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│ ↳ Policy name: [bold]policy-fail\n│ | [blue][bold]Ⓘ Advisory\n│ | [dim]This policy will fail"}, + isError: false, + }, + "unreachable": { + taskStage: func() *tfe.TaskStage { + ts := &tfe.TaskStage{} + ts.PolicyEvaluations = []*tfe.PolicyEvaluation{ + {ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{Errored: 1}, Status: "unreachable"}, + } + return ts + }, + writer: writer, + context: integrationContext, + expectedOutputs: []string{"Skipping policy evaluation."}, + isError: false, + }, + } + + for _, c := range cases { + c.writer.output.Reset() + trs := policyEvaluationSummarizer{ + cloud: b, + } + c.context.Poll(func(i int) (bool, error) { + cont, _, _ := trs.Summarize(c.context, c.writer, c.taskStage()) + if cont { + return true, nil + } + + output := c.writer.output.String() + for _, expected := range c.expectedOutputs { + if !strings.Contains(output, expected) { + t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output) + } + } + return false, nil + }) + } +} diff --git a/internal/cloud/backend_taskStages.go b/internal/cloud/backend_taskStages.go index b465af4113..3729d16602 100644 --- a/internal/cloud/backend_taskStages.go +++ b/internal/cloud/backend_taskStages.go @@ -44,7 +44,7 @@ func (b *Cloud) runTaskStages(ctx context.Context, client *tfe.Client, runId str func (b *Cloud) getTaskStageWithAllOptions(ctx *IntegrationContext, stageID string) (*tfe.TaskStage, error) { options := tfe.TaskStageReadOptions{ - Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults}, + Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults}, } stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options) if err != nil { @@ -63,13 +63,18 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr if err != nil { return err } + if s := newTaskResultSummarizer(b, ts); s != nil { summarizers = append(summarizers, s) } + if s := newPolicyEvaluationSummarizer(b, ts); s != nil { + summarizers = append(summarizers, s) + } + return ctx.Poll(func(i int) (bool, error) { options := tfe.TaskStageReadOptions{ - Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults}, + Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults}, } stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options) if err != nil { @@ -98,7 +103,25 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr errs.Append(err) } } - case "unreachable": + case tfe.TaskStageAwaitingOverride: + // TODO: Add override functionality + for _, s := range summarizers { + cont, msg, err := s.Summarize(ctx, output, stage) + if cont { + if msg != nil { + if i%4 == 0 { + if i > 0 { + output.OutputElapsed(*msg, len(*msg)) // Up to 2 digits are allowed by the max message allocation + } + } + } + return true, nil + } + if err != nil { + errs.Append(err) + } + } + case tfe.TaskStageUnreachable: return false, nil default: return false, fmt.Errorf("Invalid Task stage status: %s ", stage.Status) diff --git a/internal/cloud/testing.go b/internal/cloud/testing.go index 6cfe898337..9d2fc04452 100644 --- a/internal/cloud/testing.go +++ b/internal/cloud/testing.go @@ -209,6 +209,7 @@ func testBackend(t *testing.T, obj cty.Value) (*Cloud, func()) { b.client.CostEstimates = mc.CostEstimates b.client.Organizations = mc.Organizations b.client.Plans = mc.Plans + b.client.PolicySetOutcomes = mc.PolicySetOutcomes b.client.PolicyChecks = mc.PolicyChecks b.client.Runs = mc.Runs b.client.StateVersions = mc.StateVersions @@ -269,6 +270,7 @@ func testUnconfiguredBackend(t *testing.T) (*Cloud, func()) { b.client.CostEstimates = mc.CostEstimates b.client.Organizations = mc.Organizations b.client.Plans = mc.Plans + b.client.PolicySetOutcomes = mc.PolicySetOutcomes b.client.PolicyChecks = mc.PolicyChecks b.client.Runs = mc.Runs b.client.StateVersions = mc.StateVersions diff --git a/internal/cloud/tfe_client_mock.go b/internal/cloud/tfe_client_mock.go index 7737eb4f95..7d567833fb 100644 --- a/internal/cloud/tfe_client_mock.go +++ b/internal/cloud/tfe_client_mock.go @@ -27,6 +27,7 @@ type MockClient struct { CostEstimates *MockCostEstimates Organizations *MockOrganizations Plans *MockPlans + PolicySetOutcomes *MockPolicySetOutcomes PolicyChecks *MockPolicyChecks Runs *MockRuns StateVersions *MockStateVersions @@ -42,6 +43,7 @@ func NewMockClient() *MockClient { c.CostEstimates = newMockCostEstimates(c) c.Organizations = newMockOrganizations(c) c.Plans = newMockPlans(c) + c.PolicySetOutcomes = newMockPolicySetOutcomes(c) c.PolicyChecks = newMockPolicyChecks(c) c.Runs = newMockRuns(c) c.StateVersions = newMockStateVersions(c) @@ -545,6 +547,134 @@ func (m *MockPlans) ReadJSONOutput(ctx context.Context, planID string) ([]byte, return []byte(planOutput), nil } +type MockPolicySetOutcomes struct { + client *MockClient +} + +func newMockPolicySetOutcomes(client *MockClient) *MockPolicySetOutcomes { + return &MockPolicySetOutcomes{ + client: client, + } +} + +func (m *MockPolicySetOutcomes) List(ctx context.Context, policyEvaluationID string, options *tfe.PolicySetOutcomeListOptions) (*tfe.PolicySetOutcomeList, error) { + switch policyEvaluationID { + case "pol-pass": + return &tfe.PolicySetOutcomeList{ + Items: []*tfe.PolicySetOutcome{ + { + ID: policyEvaluationID, + Outcomes: []tfe.Outcome{ + { + EnforcementLevel: "mandatory", + Query: "data.example.rule", + Status: "passed", + PolicyName: "policy-pass", + Description: "This policy will pass", + }, + }, + Overridable: tfe.Bool(true), + Error: "", + PolicySetName: "policy-set-that-passes", + PolicySetDescription: "This policy set will always pass", + ResultCount: tfe.PolicyResultCount{ + AdvisoryFailed: 0, + MandatoryFailed: 0, + Passed: 1, + Errored: 0, + }, + }, + }, + }, nil + case "pol-fail": + return &tfe.PolicySetOutcomeList{ + Items: []*tfe.PolicySetOutcome{ + { + ID: policyEvaluationID, + Outcomes: []tfe.Outcome{ + { + EnforcementLevel: "mandatory", + Query: "data.example.rule", + Status: "failed", + PolicyName: "policy-fail", + Description: "This policy will fail", + }, + }, + Overridable: tfe.Bool(true), + Error: "", + PolicySetName: "policy-set-that-fails", + PolicySetDescription: "This policy set will always fail", + ResultCount: tfe.PolicyResultCount{ + AdvisoryFailed: 0, + MandatoryFailed: 1, + Passed: 0, + Errored: 0, + }, + }, + }, + }, nil + + case "adv-fail": + return &tfe.PolicySetOutcomeList{ + Items: []*tfe.PolicySetOutcome{ + { + ID: policyEvaluationID, + Outcomes: []tfe.Outcome{ + { + EnforcementLevel: "advisory", + Query: "data.example.rule", + Status: "failed", + PolicyName: "policy-fail", + Description: "This policy will fail", + }, + }, + Overridable: tfe.Bool(true), + Error: "", + PolicySetName: "policy-set-that-fails", + PolicySetDescription: "This policy set will always fail", + ResultCount: tfe.PolicyResultCount{ + AdvisoryFailed: 1, + MandatoryFailed: 0, + Passed: 0, + Errored: 0, + }, + }, + }, + }, nil + default: + return &tfe.PolicySetOutcomeList{ + Items: []*tfe.PolicySetOutcome{ + { + ID: policyEvaluationID, + Outcomes: []tfe.Outcome{ + { + EnforcementLevel: "mandatory", + Query: "data.example.rule", + Status: "passed", + PolicyName: "policy-pass", + Description: "This policy will pass", + }, + }, + Overridable: tfe.Bool(true), + Error: "", + PolicySetName: "policy-set-that-passes", + PolicySetDescription: "This policy set will always pass", + ResultCount: tfe.PolicyResultCount{ + AdvisoryFailed: 0, + MandatoryFailed: 0, + Passed: 1, + Errored: 0, + }, + }, + }, + }, nil + } +} + +func (m *MockPolicySetOutcomes) Read(ctx context.Context, policySetOutcomeID string) (*tfe.PolicySetOutcome, error) { + return nil, nil +} + type MockPolicyChecks struct { client *MockClient checks map[string]*tfe.PolicyCheck