feat(server): Add grpc_middleware recovery interceptor

internal/servers/controller/gateway.go

@@ -6,6 +6,7 @@ import (
 	"net"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
+	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
@@ -77,6 +78,9 @@ func newGatewayServer(
 				errorInterceptor(ctx),         // convert domain and api errors into headers for the http proxy
 				statusCodeInterceptor(ctx),    // convert grpc codes into http status codes for the http proxy (can modify the resp)
 				auditResponseInterceptor(ctx), // as we finish, audit the response
+				grpc_recovery.UnaryServerInterceptor( // recover from panics with a grpc internal error
+					grpc_recovery.WithRecoveryHandlerContext(recoveryHandler()),
+				),
 			),
 		),
 	), ticket, nil

internal/servers/controller/interceptor.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"runtime/debug"
 
+	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	"github.com/hashicorp/boundary/internal/errors"
 	pb "github.com/hashicorp/boundary/internal/gen/controller/api"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
@@ -17,9 +19,7 @@ import (
 	"github.com/hashicorp/boundary/internal/servers/controller/auth"
 	"github.com/hashicorp/boundary/internal/servers/controller/common"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
-
 	"github.com/mr-tron/base58"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
@@ -305,3 +305,17 @@ func workerRequestInfoInterceptor(ctx context.Context, eventer *event.Eventer) (
 		return handler(interceptorCtx, req)
 	}, nil
 }
+
+func recoveryHandler() grpc_recovery.RecoveryHandlerFuncContext {
+	const op = "controller.recoveryHandler"
+	return func(ctx context.Context, p interface{}) (err error) {
+		event.WriteError(
+			ctx,
+			op,
+			fmt.Errorf("recovered from panic: %v", p),
+			event.WithInfo("stack", string(debug.Stack())),
+		)
+
+		return status.Errorf(codes.Internal, "%v", p)
+	}
+}