mirror of https://github.com/hashicorp/boundary
feat: Worker Connectivity Health Check (#3650)
* feat: Worker Connectivity Health Check
Add worker upstream connection state. API calls to the worker's ops `/heath?worker_info=1` endpoint should now return a new filed called `upstream_connection_state`
```
{
"worker_process_info":{
"state":"active",
"active_session_count":0,
"upstream_connection_state":"READY"
}
}
```
The states returned will be GRPC connection states: https://grpc.github.io/grpc/core/md_doc_connectivity-semantics-and-api.html
There could be multiple upstream addresses but GRPC manual resolver determines the state of the connection.
jimlambrt-make-gen
Elim Tsiagbey
3 years ago
committed by
GitHub
parent
c55df52d44
commit
fabd4093de
@ -0,0 +1,144 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
opsservices "github.com/hashicorp/boundary/internal/gen/ops/services"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/nettest"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/grpc/resolver/manual"
|
||||
)
|
||||
|
||||
func TestMonitorUpstreamConnectionState(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stateCtx, cancelStateCtx := context.WithCancel(ctx)
|
||||
|
||||
upstreamConnectionState := new(atomic.Value)
|
||||
|
||||
servers, err := createTestServers(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
scheme := strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
res := manual.NewBuilderWithScheme(scheme)
|
||||
grpcResolver := &grpcResolverReceiver{res}
|
||||
|
||||
dialOpts := createDefaultGRPCDialOptions(res, nil)
|
||||
|
||||
cc, err := grpc.Dial(
|
||||
fmt.Sprintf("%s:///%s", res.Scheme(), servers[0].address),
|
||||
dialOpts...,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// track GRPC state changes
|
||||
go monitorUpstreamConnectionState(stateCtx, cc, upstreamConnectionState)
|
||||
|
||||
grpcResolver.InitialAddresses([]string{servers[0].address})
|
||||
|
||||
t.Cleanup(func() {
|
||||
cc.Close()
|
||||
cancelStateCtx()
|
||||
|
||||
assert.Equal(t, connectivity.Shutdown, cc.GetState())
|
||||
|
||||
for _, s := range servers {
|
||||
s.srv.GracefulStop()
|
||||
}
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
expectedResponse *opsservices.GetHealthResponse
|
||||
addresses []string
|
||||
expectedState connectivity.State
|
||||
}{
|
||||
{
|
||||
name: "connection with 1 good address",
|
||||
addresses: []string{servers[0].address},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
{
|
||||
name: "connection with multiple good addresses",
|
||||
addresses: []string{servers[1].address, servers[2].address},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
{
|
||||
name: "connection with bad address",
|
||||
addresses: []string{"bad_address"},
|
||||
expectedState: connectivity.TransientFailure,
|
||||
},
|
||||
{
|
||||
name: "connection with 1 bad address and 1 good address",
|
||||
addresses: []string{servers[0].address, "bad_address"},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
doneWait := make(chan struct{})
|
||||
grpcResolver.SetAddresses(tt.addresses)
|
||||
|
||||
go waitForConnectionStateCondition(upstreamConnectionState, tt.expectedState, doneWait)
|
||||
|
||||
select {
|
||||
case <-doneWait:
|
||||
// The connection condition was met, Proceed with assertions
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Error("Time out waiting for condition")
|
||||
}
|
||||
|
||||
got := upstreamConnectionState.Load()
|
||||
assert.Equal(t, tt.expectedState, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func waitForConnectionStateCondition(upstreamConnectionState *atomic.Value, expectedValue connectivity.State, ch chan<- struct{}) {
|
||||
for {
|
||||
currentValue := upstreamConnectionState.Load()
|
||||
if expectedValue == currentValue {
|
||||
ch <- struct{}{}
|
||||
return
|
||||
}
|
||||
// Small delay for checking state
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type serverTestInfo struct {
|
||||
srv *grpc.Server
|
||||
address string
|
||||
}
|
||||
|
||||
func createTestServers(t *testing.T) ([]*serverTestInfo, error) {
|
||||
serverCount := 4
|
||||
|
||||
servers := make([]*serverTestInfo, 0, serverCount)
|
||||
for i := 0; i < serverCount; i++ {
|
||||
listener, err := nettest.NewLocalListener("tcp")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
lInfo := &serverTestInfo{srv: srv, address: listener.Addr().String()}
|
||||
servers = append(servers, lInfo)
|
||||
go func(i int) {
|
||||
servers[i].srv.Serve(listener)
|
||||
}(i)
|
||||
}
|
||||
|
||||
return servers, nil
|
||||
}
|
||||
Loading…
Reference in new issue