diff --git a/internal/servers/controller/internal/metric/api.go b/internal/servers/controller/internal/metric/api.go index 74432c8353..7c2a4f5c25 100644 --- a/internal/servers/controller/internal/metric/api.go +++ b/internal/servers/controller/internal/metric/api.go @@ -198,7 +198,7 @@ var expectedStatusCodesPerMethod = map[string][]int{ }, } -// pathLabel maps the requested path the the label value recorded for metrics +// pathLabel maps the requested path to the label value recorded for metrics func pathLabel(incomingPath string) string { if incomingPath == "" || incomingPath[0] != '/' { incomingPath = fmt.Sprintf("/%s", incomingPath) diff --git a/internal/servers/worker/handler.go b/internal/servers/worker/handler.go index 045253b01c..ae26217242 100644 --- a/internal/servers/worker/handler.go +++ b/internal/servers/worker/handler.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/boundary/internal/observability/event" "github.com/hashicorp/boundary/internal/proxy" "github.com/hashicorp/boundary/internal/servers/common" + "github.com/hashicorp/boundary/internal/servers/worker/internal/metric" proxyHandlers "github.com/hashicorp/boundary/internal/servers/worker/proxy" "github.com/hashicorp/boundary/internal/servers/worker/session" "github.com/hashicorp/go-secure-stdlib/listenerutil" @@ -40,8 +41,8 @@ func (w *Worker) handler(props HandlerProperties) (http.Handler, error) { mux.Handle("/v1/proxy", h) genericWrappedHandler := w.wrapGenericHandler(mux, props) - - return genericWrappedHandler, nil + metricHandler := metric.InstrumentProxyHttpHandler(genericWrappedHandler) + return metricHandler, nil } func (w *Worker) handleProxy(listenerCfg *listenerutil.ListenerConfig) (http.HandlerFunc, error) { diff --git a/internal/servers/worker/internal/metric/proxy_http.go b/internal/servers/worker/internal/metric/proxy_http.go new file mode 100644 index 0000000000..22578ded93 --- /dev/null +++ b/internal/servers/worker/internal/metric/proxy_http.go @@ -0,0 +1,102 @@ +// Package metric provides functions to initialize the worker specific +// collectors and hooks to measure metrics and update the relevant collectors. +package metric + +import ( + "fmt" + "net/http" + "path" + "strconv" + "strings" + + "github.com/hashicorp/boundary/globals" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + invalidPathValue = "invalid" + proxyPathValue = "/v1/proxy" + + labelHttpCode = "code" + labelHttpPath = "path" + labelHttpMethod = "method" + proxySubSystem = "worker_proxy" +) + +// httpTimeUntilHeader collects measurements of how long it takes +// the boundary system to hijack an HTTP request into a websocket connection +// for the proxy worker. +var httpTimeUntilHeader prometheus.ObserverVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: globals.MetricNamespace, + Subsystem: proxySubSystem, + Name: "http_write_header_duration_seconds", + Help: "Histogram of time elapsed after the TLS connection is established to when the first http header is written back from the server.", + Buckets: prometheus.DefBuckets, + }, + []string{labelHttpCode, labelHttpPath, labelHttpMethod}, +) + +var expectedHttpErrCodes = []int{ + http.StatusUpgradeRequired, + http.StatusMethodNotAllowed, + http.StatusBadRequest, + http.StatusForbidden, + http.StatusNotImplemented, + http.StatusSwitchingProtocols, + http.StatusInternalServerError, +} + +// pathLabel maps the requested path to the label value recorded for metric +func pathLabel(incomingPath string) string { + if incomingPath == "" || incomingPath[0] != '/' { + incomingPath = fmt.Sprintf("/%s", incomingPath) + } + incomingPath = path.Clean(incomingPath) + + if incomingPath == proxyPathValue { + return proxyPathValue + } + return invalidPathValue +} + +// InstrumentProxyHttpHandler provides a proxy handler which measures +// time until header is returned form the server and attaches status code, +// method, and path labels for each of these measurements. +func InstrumentProxyHttpHandler(wrapped http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + l := prometheus.Labels{ + labelHttpPath: pathLabel(req.URL.Path), + } + promhttp.InstrumentHandlerTimeToWriteHeader( + httpTimeUntilHeader.MustCurryWith(l), + wrapped, + ).ServeHTTP(rw, req) + }) +} + +// InstrumentProxyHttpCollectors registers the proxy collectors to the default +// prometheus register and initializes them to 0 for all possible label +// combinations. +func InstrumentProxyHttpCollectors(r prometheus.Registerer) { + if r == nil { + return + } + r.MustRegister(httpTimeUntilHeader) + + p := proxyPathValue + method := http.MethodGet + for _, sc := range expectedHttpErrCodes { + l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(method)} + httpTimeUntilHeader.With(l) + } + + // When an invalid path is found, any method is possible, but we expect + // an error response. + p = invalidPathValue + for _, sc := range []int{http.StatusNotFound, http.StatusMethodNotAllowed} { + l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(method)} + httpTimeUntilHeader.With(l) + } +} diff --git a/internal/servers/worker/testing.go b/internal/servers/worker/testing.go index 4775ed7a33..71b54ea5d1 100644 --- a/internal/servers/worker/testing.go +++ b/internal/servers/worker/testing.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/go-hclog" wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-secure-stdlib/base62" + "github.com/prometheus/client_golang/prometheus" ) // TestWorker wraps a base.Server and Worker to provide a @@ -179,6 +180,10 @@ type TestWorkerOpts struct { // The logger to use, or one will be created Logger hclog.Logger + // The registerer to use for registering all the collectors. Nil means + // no metrics are registered. + PrometheusRegisterer prometheus.Registerer + // The amount of time to wait before marking connections as closed when a // connection cannot be made back to the controller StatusGracePeriodDuration time.Duration @@ -230,6 +235,8 @@ func NewTestWorker(t *testing.T, opts *TestWorkerOpts) *TestWorker { }) } + tw.b.PrometheusRegisterer = opts.PrometheusRegisterer + // Initialize status grace period tw.b.SetStatusGracePeriodDuration(opts.StatusGracePeriodDuration) diff --git a/internal/servers/worker/worker.go b/internal/servers/worker/worker.go index c9659918a1..332b017aba 100644 --- a/internal/servers/worker/worker.go +++ b/internal/servers/worker/worker.go @@ -19,6 +19,7 @@ import ( pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" "github.com/hashicorp/boundary/internal/observability/event" "github.com/hashicorp/boundary/internal/servers" + "github.com/hashicorp/boundary/internal/servers/worker/internal/metric" "github.com/hashicorp/boundary/internal/servers/worker/session" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/base62" @@ -64,6 +65,7 @@ type Worker struct { } func New(conf *Config) (*Worker, error) { + metric.InstrumentProxyHttpCollectors(conf.PrometheusRegisterer) w := &Worker{ conf: conf, logger: conf.Logger.Named("worker"),