Initialize and provide handler for proxy worker http metrics (#1961)

*feat(metrics): implement prometheus handler for proxy worker http requests and record metric for header write duration

internal/servers/controller/internal/metric/api.go

@@ -198,7 +198,7 @@ var expectedStatusCodesPerMethod = map[string][]int{
 	},
 }
 
-// pathLabel maps the requested path the the label value recorded for metrics
+// pathLabel maps the requested path to the label value recorded for metrics
 func pathLabel(incomingPath string) string {
 	if incomingPath == "" || incomingPath[0] != '/' {
 		incomingPath = fmt.Sprintf("/%s", incomingPath)

internal/servers/worker/handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/proxy"
 	"github.com/hashicorp/boundary/internal/servers/common"
+	"github.com/hashicorp/boundary/internal/servers/worker/internal/metric"
 	proxyHandlers "github.com/hashicorp/boundary/internal/servers/worker/proxy"
 	"github.com/hashicorp/boundary/internal/servers/worker/session"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
@@ -40,8 +41,8 @@ func (w *Worker) handler(props HandlerProperties) (http.Handler, error) {
 	mux.Handle("/v1/proxy", h)
 
 	genericWrappedHandler := w.wrapGenericHandler(mux, props)
-
-	return genericWrappedHandler, nil
+	metricHandler := metric.InstrumentProxyHttpHandler(genericWrappedHandler)
+	return metricHandler, nil
 }
 
 func (w *Worker) handleProxy(listenerCfg *listenerutil.ListenerConfig) (http.HandlerFunc, error) {

internal/servers/worker/internal/metric/proxy_http.go

@@ -0,0 +1,102 @@
+// Package metric provides functions to initialize the worker specific
+// collectors and hooks to measure metrics and update the relevant collectors.
+package metric
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/boundary/globals"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+	invalidPathValue = "invalid"
+	proxyPathValue   = "/v1/proxy"
+
+	labelHttpCode   = "code"
+	labelHttpPath   = "path"
+	labelHttpMethod = "method"
+	proxySubSystem  = "worker_proxy"
+)
+
+// httpTimeUntilHeader collects measurements of how long it takes
+// the boundary system to hijack an HTTP request into a websocket connection
+// for the proxy worker.
+var httpTimeUntilHeader prometheus.ObserverVec = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Namespace: globals.MetricNamespace,
+		Subsystem: proxySubSystem,
+		Name:      "http_write_header_duration_seconds",
+		Help:      "Histogram of time elapsed after the TLS connection is established to when the first http header is written back from the server.",
+		Buckets:   prometheus.DefBuckets,
+	},
+	[]string{labelHttpCode, labelHttpPath, labelHttpMethod},
+)
+
+var expectedHttpErrCodes = []int{
+	http.StatusUpgradeRequired,
+	http.StatusMethodNotAllowed,
+	http.StatusBadRequest,
+	http.StatusForbidden,
+	http.StatusNotImplemented,
+	http.StatusSwitchingProtocols,
+	http.StatusInternalServerError,
+}
+
+// pathLabel maps the requested path to the label value recorded for metric
+func pathLabel(incomingPath string) string {
+	if incomingPath == "" || incomingPath[0] != '/' {
+		incomingPath = fmt.Sprintf("/%s", incomingPath)
+	}
+	incomingPath = path.Clean(incomingPath)
+
+	if incomingPath == proxyPathValue {
+		return proxyPathValue
+	}
+	return invalidPathValue
+}
+
+// InstrumentProxyHttpHandler provides a proxy handler which measures
+// time until header is returned form the server and attaches status code,
+// method, and path labels for each of these measurements.
+func InstrumentProxyHttpHandler(wrapped http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		l := prometheus.Labels{
+			labelHttpPath: pathLabel(req.URL.Path),
+		}
+		promhttp.InstrumentHandlerTimeToWriteHeader(
+			httpTimeUntilHeader.MustCurryWith(l),
+			wrapped,
+		).ServeHTTP(rw, req)
+	})
+}
+
+// InstrumentProxyHttpCollectors registers the proxy collectors to the default
+// prometheus register and initializes them to 0 for all possible label
+// combinations.
+func InstrumentProxyHttpCollectors(r prometheus.Registerer) {
+	if r == nil {
+		return
+	}
+	r.MustRegister(httpTimeUntilHeader)
+
+	p := proxyPathValue
+	method := http.MethodGet
+	for _, sc := range expectedHttpErrCodes {
+		l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(method)}
+		httpTimeUntilHeader.With(l)
+	}
+
+	// When an invalid path is found, any method is possible, but we expect
+	// an error response.
+	p = invalidPathValue
+	for _, sc := range []int{http.StatusNotFound, http.StatusMethodNotAllowed} {
+		l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(method)}
+		httpTimeUntilHeader.With(l)
+	}
+}

internal/servers/worker/testing.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/hashicorp/go-secure-stdlib/base62"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // TestWorker wraps a base.Server and Worker to provide a
@@ -179,6 +180,10 @@ type TestWorkerOpts struct {
 	// The logger to use, or one will be created
 	Logger hclog.Logger
 
+	// The registerer to use for registering all the collectors.  Nil means
+	// no metrics are registered.
+	PrometheusRegisterer prometheus.Registerer
+
 	// The amount of time to wait before marking connections as closed when a
 	// connection cannot be made back to the controller
 	StatusGracePeriodDuration time.Duration
@@ -230,6 +235,8 @@ func NewTestWorker(t *testing.T, opts *TestWorkerOpts) *TestWorker {
 		})
 	}
 
+	tw.b.PrometheusRegisterer = opts.PrometheusRegisterer
+
 	// Initialize status grace period
 	tw.b.SetStatusGracePeriodDuration(opts.StatusGracePeriodDuration)
 

internal/servers/worker/worker.go

@@ -19,6 +19,7 @@ import (
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/servers/worker/internal/metric"
 	"github.com/hashicorp/boundary/internal/servers/worker/session"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/base62"
@@ -64,6 +65,7 @@ type Worker struct {
 }
 
 func New(conf *Config) (*Worker, error) {
+	metric.InstrumentProxyHttpCollectors(conf.PrometheusRegisterer)
 	w := &Worker{
 		conf:                  conf,
 		logger:                conf.Logger.Named("worker"),