aherman
/
boundary
mirror of https://github.com/hashicorp/boundary
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Backport of feat(pprof): Enable pprof endpoints behind flag into release/0.19.x (#6646)

Browse Source 
* no-op commit due to failed cherry-picking

* feat(pprof): Enable pprof endpoints behind flag (#6644)

---------

Co-authored-by: temp <temp@hashicorp.com>
Co-authored-by: Louis Ruch <louisruch@gmail.com>
pull/6658/head
hc-github-team-secure-boundary 3 weeks ago committed by GitHub
parent 60a0fd2f1c
commit 95d6b8f550
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 67 additions and 78 deletions
Show Stats Download Patch File Download Diff File

7
Makefile
Unescape View File

@ -66,13 +66,6 @@ install: build
install-no-plugins: export SKIP_PLUGIN_BUILD=1
install-no-plugins: install
.PHONY: build-pprof
build-pprof: BUILD_TAGS+=pprof
build-pprof: BUILD_TAGS+=ui
build-pprof:
@echo "==> Building Boundary with memory pprof enabled"
@CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/build.sh'"
.PHONY: build-memprof
build-memprof: BUILD_TAGS+=memprofiler
build-memprof:

14
internal/cmd/base/pprof_off.go
Unescape View File

@ -1,14 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
//go:build !pprof
// +build !pprof
package base
import (
"context"
)
func StartPprof(_ context.Context) {
}

39
internal/cmd/base/pprof_on.go
Unescape View File

@ -1,39 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
//go:build pprof
// +build pprof
package base
import (
"context"
"errors"
"net/http"
"net/http/pprof"
"github.com/hashicorp/boundary/internal/event"
)
func StartPprof(ctx context.Context) {
const op = "base.StartPprof"
go func() {
const addr = "localhost:6060"
mux := http.NewServeMux()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
mux.Handle("/debug/pprof/block", pprof.Handler("block"))
mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
event.WriteSysEvent(ctx, op, "starting pprof HTTP server", "addr", addr)
if err := http.ListenAndServe(addr, mux); err != nil && !errors.Is(err, http.ErrServerClosed) {
event.WriteSysEvent(ctx, op, "failed to serve pprof HTTP server", "error", err.Error())
}
}()
}

12
internal/cmd/commands/dev/dev.go
Unescape View File

@ -87,6 +87,7 @@ type Command struct {
flagWorkerProxyListenAddr string
flagWorkerPublicAddr string
flagOpsListenAddr string
flagDebug bool
flagUiPassthroughDir string
flagRecoveryKey string
flagDatabaseUrl string
@ -262,6 +263,13 @@ func (c *Command) Flags() *base.FlagSets {
Usage: "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.",
})
f.BoolVar(&base.BoolVar{
Name: "debug",
Target: &c.flagDebug,
Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.",
Hidden: true,
})
f.BoolVar(&base.BoolVar{
Name: "controller-only",
Target: &c.flagControllerOnly,
@ -722,8 +730,6 @@ func (c *Command) Run(args []string) int {
return base.CommandCliError
}
base.StartPprof(c.Context)
if c.flagRecoveryKey != "" {
c.Config.DevRecoveryKey = c.flagRecoveryKey
}
@ -993,7 +999,7 @@ func (c *Command) Run(args []string) int {
return base.CommandCliError
}
opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...)
opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...)
if err != nil {
c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error())
return base.CommandCliError

11
internal/cmd/commands/server/server.go
Unescape View File

@ -62,6 +62,7 @@ type Command struct {
flagLogLevel string
flagLogFormat string
flagCombineLogs bool
flagDebug bool
flagSkipPlugins bool
flagSkipAliasTargetCreation bool
flagWorkerDnsServer string
@ -148,7 +149,12 @@ func (c *Command) Flags() *base.FlagSets {
Target: &c.flagWorkerAuthCaReinitialize,
Hidden: true,
})
f.BoolVar(&base.BoolVar{
Name: "debug",
Target: &c.flagDebug,
Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.",
Hidden: true,
})
f.BoolVar(&base.BoolVar{
Name: "skip-plugins",
Target: &c.flagSkipPlugins,
@ -217,7 +223,6 @@ func (c *Command) Run(args []string) int {
c.WorkerAuthDebuggingEnabled.Store(c.Config.EnableWorkerAuthDebugging)
base.StartMemProfiler(c.Context)
base.StartPprof(c.Context)
// Note: the checks directly after this must remain where they are because
// they rely on the state of configured KMSes.
@ -548,7 +553,7 @@ func (c *Command) Run(args []string) int {
return base.CommandCliError
}
opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...)
opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...)
if err != nil {
c.UI.Error(err.Error())
return base.CommandCliError

21
internal/cmd/ops/server.go
Unescape View File

@ -11,6 +11,7 @@ import (
"fmt"
"net"
"net/http"
"net/http/pprof"
"os"
"time"
@ -39,7 +40,7 @@ type opsBundle struct {
// NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers.
// If Controller is set-up, NewServer will set-up a health endpoint for it.
func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, listeners ...*base.ServerListener) (*Server, error) {
func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, enableDebug bool, listeners ...*base.ServerListener) (*Server, error) {
const op = "ops.NewServer()"
if l == nil {
return nil, fmt.Errorf("%s: missing logger", op)
@ -57,7 +58,7 @@ func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w
return nil, fmt.Errorf("%s: missing ops listener", op)
}
h, err := createOpsHandler(ctx, ln.Config, c, w)
h, err := createOpsHandler(ctx, ln.Config, c, w, enableDebug)
if err != nil {
return nil, err
}
@ -131,7 +132,7 @@ func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) {
<-time.After(d)
}
func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker) (http.Handler, error) {
func createOpsHandler(_ context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker, enableDebug bool) (http.Handler, error) {
mux := http.NewServeMux()
var h http.Handler
var err error
@ -156,6 +157,20 @@ func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c
mux.Handle("/health", h)
}
mux.Handle("/metrics", promhttp.Handler())
if enableDebug {
// Turn on pprof endpoints if debug is enabled.
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
mux.Handle("/debug/pprof/block", pprof.Handler("block"))
mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
}
return cleanhttp.PrintablePathCheckHandler(mux, nil), nil
}

31
internal/cmd/ops/server_test.go
Unescape View File

@ -106,7 +106,7 @@ func TestNewServer(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, tt.listeners...)
s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, false, tt.listeners...)
if tt.expErr {
require.EqualError(t, err, tt.expErrMsg)
require.Nil(t, s)
@ -297,7 +297,7 @@ func TestNewServerIntegration(t *testing.T) {
err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"})
require.NoError(t, err)
s, err := NewServer(context.Background(), hclog.Default(), nil, nil, bs.Listeners...)
s, err := NewServer(context.Background(), hclog.Default(), nil, nil, false, bs.Listeners...)
if tt.expErr {
require.EqualError(t, err, tt.expErrMsg)
require.Nil(t, s)
@ -597,7 +597,7 @@ func TestHealthEndpointLifecycle(t *testing.T) {
require.NoError(t, err)
// Controller has started and is set onto our Command object, start ops.
opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, tc.Config().Listeners...)
opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, false, tc.Config().Listeners...)
require.NoError(t, err)
opsServer.Start()
@ -692,6 +692,7 @@ func TestCreateOpsHandler(t *testing.T) {
name string
setupController bool
setupWorker bool
enableDebug bool
lncfg *listenerutil.ListenerConfig
expErr bool
expErrMsg string
@ -800,6 +801,28 @@ func TestCreateOpsHandler(t *testing.T) {
expErr: true,
expErrMsg: "controller.(Controller).GetHealthHandler: received nil listener config",
},
{
name: "pprof disabled by debug flag",
enableDebug: false,
lncfg: &listenerutil.ListenerConfig{},
assertions: func(t *testing.T, addr string) {
rsp, err := http.Get("http://" + addr + "/debug/pprof/")
require.NoError(t, err)
require.Equal(t, http.StatusNotFound, rsp.StatusCode)
require.NoError(t, rsp.Body.Close())
},
},
{
name: "pprof enabled by debug flag",
enableDebug: true,
lncfg: &listenerutil.ListenerConfig{},
assertions: func(t *testing.T, addr string) {
rsp, err := http.Get("http://" + addr + "/debug/pprof/")
require.NoError(t, err)
require.Equal(t, http.StatusOK, rsp.StatusCode)
require.NoError(t, rsp.Body.Close())
},
},
}
for _, tt := range tests {
@ -816,7 +839,7 @@ func TestCreateOpsHandler(t *testing.T) {
w = tc.Worker()
}
h, err := createOpsHandler(ctx, tt.lncfg, c, w)
h, err := createOpsHandler(ctx, tt.lncfg, c, w, tt.enableDebug)
if tt.expErr {
require.EqualError(t, err, tt.expErrMsg)
require.Nil(t, h)

10
testing/TRACING.md
Unescape View File

@ -1,19 +1,19 @@
# Tracing in Boundary
Boundary includes a small number of runtime tracing user regions, which can be used to see where Boundary spends its time during execution.
To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint is as simple as building with the `pprof` build tag or running
To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint requires enabling the runtime `-debug` flag on a process with an `ops` listener.
```
make build-pprof
make build
boundary dev -debug -ops-listen-address=127.0.0.1:9203
```
This will create a new HTTP endpoint on `localhost:6060` of the running binary. As such, it's only accessible to the users on the same machine.
Remember to remove this code again once you're done testing.
This will expose the pprof endpoints on the configured ops listener. With the example above, that means `127.0.0.1:9203`, so it's only accessible to users on the same machine.
To create a trace, one can use any tool that allows creating HTTP requests, e.g. `curl`. To create a 3 second trace:
```
$ curl -o trace.out http://localhost:6060/debug/pprof/trace?seconds=3
$ curl -o trace.out http://127.0.0.1:9203/debug/pprof/trace?seconds=3
```
Traces are most interesting if they contain some request handling, so it is recommended to prepare some HTTP requests that trigger the behavior you want to understand that you can run while the trace is being collected.

Write Preview
Loading…
Cancel
Save