diff --git a/Makefile b/Makefile index 7adbaeac69..495dbafab4 100644 --- a/Makefile +++ b/Makefile @@ -66,13 +66,6 @@ install: build install-no-plugins: export SKIP_PLUGIN_BUILD=1 install-no-plugins: install -.PHONY: build-pprof -build-pprof: BUILD_TAGS+=pprof -build-pprof: BUILD_TAGS+=ui -build-pprof: - @echo "==> Building Boundary with memory pprof enabled" - @CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/build.sh'" - .PHONY: build-memprof build-memprof: BUILD_TAGS+=memprofiler build-memprof: diff --git a/internal/cmd/base/pprof_off.go b/internal/cmd/base/pprof_off.go deleted file mode 100644 index e8e9cbb5bb..0000000000 --- a/internal/cmd/base/pprof_off.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -//go:build !pprof -// +build !pprof - -package base - -import ( - "context" -) - -func StartPprof(_ context.Context) { -} diff --git a/internal/cmd/base/pprof_on.go b/internal/cmd/base/pprof_on.go deleted file mode 100644 index 3a23395fdf..0000000000 --- a/internal/cmd/base/pprof_on.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -//go:build pprof -// +build pprof - -package base - -import ( - "context" - "errors" - "net/http" - "net/http/pprof" - - "github.com/hashicorp/boundary/internal/event" -) - -func StartPprof(ctx context.Context) { - const op = "base.StartPprof" - go func() { - const addr = "localhost:6060" - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - mux.Handle("/debug/pprof/block", pprof.Handler("block")) - mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) - mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) - event.WriteSysEvent(ctx, op, "starting pprof HTTP server", "addr", addr) - if err := http.ListenAndServe(addr, mux); err != nil && !errors.Is(err, http.ErrServerClosed) { - event.WriteSysEvent(ctx, op, "failed to serve pprof HTTP server", "error", err.Error()) - } - }() -} diff --git a/internal/cmd/commands/dev/dev.go b/internal/cmd/commands/dev/dev.go index 803fb5a03a..4621fdf37a 100644 --- a/internal/cmd/commands/dev/dev.go +++ b/internal/cmd/commands/dev/dev.go @@ -87,6 +87,7 @@ type Command struct { flagWorkerProxyListenAddr string flagWorkerPublicAddr string flagOpsListenAddr string + flagDebug bool flagUiPassthroughDir string flagRecoveryKey string flagDatabaseUrl string @@ -262,6 +263,13 @@ func (c *Command) Flags() *base.FlagSets { Usage: "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.", }) + f.BoolVar(&base.BoolVar{ + Name: "debug", + Target: &c.flagDebug, + Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.", + Hidden: true, + }) + f.BoolVar(&base.BoolVar{ Name: "controller-only", Target: &c.flagControllerOnly, @@ -722,8 +730,6 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - base.StartPprof(c.Context) - if c.flagRecoveryKey != "" { c.Config.DevRecoveryKey = c.flagRecoveryKey } @@ -993,7 +999,7 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...) + opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...) if err != nil { c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error()) return base.CommandCliError diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go index b9d8bee85f..1803b091a4 100644 --- a/internal/cmd/commands/server/server.go +++ b/internal/cmd/commands/server/server.go @@ -62,6 +62,7 @@ type Command struct { flagLogLevel string flagLogFormat string flagCombineLogs bool + flagDebug bool flagSkipPlugins bool flagSkipAliasTargetCreation bool flagWorkerDnsServer string @@ -148,7 +149,12 @@ func (c *Command) Flags() *base.FlagSets { Target: &c.flagWorkerAuthCaReinitialize, Hidden: true, }) - + f.BoolVar(&base.BoolVar{ + Name: "debug", + Target: &c.flagDebug, + Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.", + Hidden: true, + }) f.BoolVar(&base.BoolVar{ Name: "skip-plugins", Target: &c.flagSkipPlugins, @@ -217,7 +223,6 @@ func (c *Command) Run(args []string) int { c.WorkerAuthDebuggingEnabled.Store(c.Config.EnableWorkerAuthDebugging) base.StartMemProfiler(c.Context) - base.StartPprof(c.Context) // Note: the checks directly after this must remain where they are because // they rely on the state of configured KMSes. @@ -548,7 +553,7 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...) + opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...) if err != nil { c.UI.Error(err.Error()) return base.CommandCliError diff --git a/internal/cmd/ops/server.go b/internal/cmd/ops/server.go index 75a69e0448..a2258ebbd1 100644 --- a/internal/cmd/ops/server.go +++ b/internal/cmd/ops/server.go @@ -11,6 +11,7 @@ import ( "fmt" "net" "net/http" + "net/http/pprof" "os" "time" @@ -39,7 +40,7 @@ type opsBundle struct { // NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers. // If Controller is set-up, NewServer will set-up a health endpoint for it. -func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, listeners ...*base.ServerListener) (*Server, error) { +func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, enableDebug bool, listeners ...*base.ServerListener) (*Server, error) { const op = "ops.NewServer()" if l == nil { return nil, fmt.Errorf("%s: missing logger", op) @@ -57,7 +58,7 @@ func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w return nil, fmt.Errorf("%s: missing ops listener", op) } - h, err := createOpsHandler(ctx, ln.Config, c, w) + h, err := createOpsHandler(ctx, ln.Config, c, w, enableDebug) if err != nil { return nil, err } @@ -131,7 +132,7 @@ func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) { <-time.After(d) } -func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker) (http.Handler, error) { +func createOpsHandler(_ context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker, enableDebug bool) (http.Handler, error) { mux := http.NewServeMux() var h http.Handler var err error @@ -156,6 +157,20 @@ func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c mux.Handle("/health", h) } mux.Handle("/metrics", promhttp.Handler()) + if enableDebug { + // Turn on pprof endpoints if debug is enabled. + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.Handle("/debug/pprof/block", pprof.Handler("block")) + mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) + mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + } return cleanhttp.PrintablePathCheckHandler(mux, nil), nil } diff --git a/internal/cmd/ops/server_test.go b/internal/cmd/ops/server_test.go index 65cbc03606..6f7b5f9bc1 100644 --- a/internal/cmd/ops/server_test.go +++ b/internal/cmd/ops/server_test.go @@ -106,7 +106,7 @@ func TestNewServer(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, tt.listeners...) + s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, false, tt.listeners...) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, s) @@ -297,7 +297,7 @@ func TestNewServerIntegration(t *testing.T) { err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"}) require.NoError(t, err) - s, err := NewServer(context.Background(), hclog.Default(), nil, nil, bs.Listeners...) + s, err := NewServer(context.Background(), hclog.Default(), nil, nil, false, bs.Listeners...) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, s) @@ -597,7 +597,7 @@ func TestHealthEndpointLifecycle(t *testing.T) { require.NoError(t, err) // Controller has started and is set onto our Command object, start ops. - opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, tc.Config().Listeners...) + opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, false, tc.Config().Listeners...) require.NoError(t, err) opsServer.Start() @@ -692,6 +692,7 @@ func TestCreateOpsHandler(t *testing.T) { name string setupController bool setupWorker bool + enableDebug bool lncfg *listenerutil.ListenerConfig expErr bool expErrMsg string @@ -800,6 +801,28 @@ func TestCreateOpsHandler(t *testing.T) { expErr: true, expErrMsg: "controller.(Controller).GetHealthHandler: received nil listener config", }, + { + name: "pprof disabled by debug flag", + enableDebug: false, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/debug/pprof/") + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + require.NoError(t, rsp.Body.Close()) + }, + }, + { + name: "pprof enabled by debug flag", + enableDebug: true, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/debug/pprof/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.NoError(t, rsp.Body.Close()) + }, + }, } for _, tt := range tests { @@ -816,7 +839,7 @@ func TestCreateOpsHandler(t *testing.T) { w = tc.Worker() } - h, err := createOpsHandler(ctx, tt.lncfg, c, w) + h, err := createOpsHandler(ctx, tt.lncfg, c, w, tt.enableDebug) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, h) diff --git a/testing/TRACING.md b/testing/TRACING.md index f9fa37a32f..71550b4e47 100644 --- a/testing/TRACING.md +++ b/testing/TRACING.md @@ -1,19 +1,19 @@ # Tracing in Boundary Boundary includes a small number of runtime tracing user regions, which can be used to see where Boundary spends its time during execution. -To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint is as simple as building with the `pprof` build tag or running +To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint requires enabling the runtime `-debug` flag on a process with an `ops` listener. ``` -make build-pprof +make build +boundary dev -debug -ops-listen-address=127.0.0.1:9203 ``` -This will create a new HTTP endpoint on `localhost:6060` of the running binary. As such, it's only accessible to the users on the same machine. -Remember to remove this code again once you're done testing. +This will expose the pprof endpoints on the configured ops listener. With the example above, that means `127.0.0.1:9203`, so it's only accessible to users on the same machine. To create a trace, one can use any tool that allows creating HTTP requests, e.g. `curl`. To create a 3 second trace: ``` -$ curl -o trace.out http://localhost:6060/debug/pprof/trace?seconds=3 +$ curl -o trace.out http://127.0.0.1:9203/debug/pprof/trace?seconds=3 ``` Traces are most interesting if they contain some request handling, so it is recommended to prepare some HTTP requests that trigger the behavior you want to understand that you can run while the trace is being collected.