From e07792cf747917ee6776bfa2c359180c54876025 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Mon, 13 Sep 2021 20:26:54 -0400 Subject: [PATCH] Add endpoint preferencer (#1526) --- go.mod | 3 +- go.sum | 1 - internal/libs/endpoint/doc.go | 10 ++ internal/libs/endpoint/matcher.go | 48 +++++++++ internal/libs/endpoint/matcher_test.go | 48 +++++++++ internal/libs/endpoint/option.go | 106 ++++++++++++++++++++ internal/libs/endpoint/options_test.go | 79 +++++++++++++++ internal/libs/endpoint/preferencer.go | 76 ++++++++++++++ internal/libs/endpoint/preferencer_test.go | 110 +++++++++++++++++++++ 9 files changed, 479 insertions(+), 2 deletions(-) create mode 100644 internal/libs/endpoint/doc.go create mode 100644 internal/libs/endpoint/matcher.go create mode 100644 internal/libs/endpoint/matcher_test.go create mode 100644 internal/libs/endpoint/option.go create mode 100644 internal/libs/endpoint/options_test.go create mode 100644 internal/libs/endpoint/preferencer.go create mode 100644 internal/libs/endpoint/preferencer_test.go diff --git a/go.mod b/go.mod index 64c175c50c..d9648d719a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ replace github.com/hashicorp/boundary/sdk => ./sdk require ( github.com/armon/go-metrics v0.3.9 github.com/bufbuild/buf v0.37.0 - github.com/dhui/dktest v0.3.4 + github.com/dhui/dktest v0.3.4 // indirect github.com/fatih/color v1.12.0 github.com/fatih/structs v1.1.0 github.com/favadi/protoc-go-inject-tag v1.1.0 @@ -68,6 +68,7 @@ require ( github.com/pires/go-proxyproto v0.5.0 github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 + github.com/ryanuber/go-glob v1.0.0 github.com/spf13/cobra v1.1.1 // indirect github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 diff --git a/go.sum b/go.sum index 9e64763ba9..021d93481c 100644 --- a/go.sum +++ b/go.sum @@ -380,7 +380,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= diff --git a/internal/libs/endpoint/doc.go b/internal/libs/endpoint/doc.go new file mode 100644 index 0000000000..74bcbd7405 --- /dev/null +++ b/internal/libs/endpoint/doc.go @@ -0,0 +1,10 @@ +package endpoint + +// This package contains enpoint-related libraries. +// +// Currently, this consists only of a preference chooser that, given inputs of +// IP addresses/DNS names and a user-defined preference string, can select the +// most preferred endpoint to use. If no user-defined preference string is +// supplied, an endpoint is selected at random. Creating a preferencer will +// validate input, so calling NewPreferencer and ignoring the returned struct is +// a fine way to validate incoming preference order statements. diff --git a/internal/libs/endpoint/matcher.go b/internal/libs/endpoint/matcher.go new file mode 100644 index 0000000000..4f8deb79aa --- /dev/null +++ b/internal/libs/endpoint/matcher.go @@ -0,0 +1,48 @@ +package endpoint + +import ( + "net" + + "github.com/ryanuber/go-glob" +) + +// matcher is a function that given an input returns whether there is a match +type matcher interface { + Match(string) bool +} + +var ( + _ matcher = (*dnsMatcher)(nil) + _ matcher = (*cidrMatcher)(nil) +) + +// DnsMatcher is a function that given an input returns true if there is a +// globbed DNS match +type dnsMatcher struct { + pattern string +} + +// Match satisfies the matcher interface +func (m dnsMatcher) Match(in string) bool { + return glob.Glob(m.pattern, in) +} + +// cidrMatcher is a function that given an input returns true if the input is +// contained within the given net +type cidrMatcher struct { + ipNet *net.IPNet +} + +// Match satisfies the matcher interface +func (m cidrMatcher) Match(in string) bool { + // Can't be created directly since this is unexported so this should never + // actually trigger, but for safety + if m.ipNet == nil { + return false + } + ip := net.ParseIP(in) + if ip == nil { + return false + } + return m.ipNet.Contains(ip) +} diff --git a/internal/libs/endpoint/matcher_test.go b/internal/libs/endpoint/matcher_test.go new file mode 100644 index 0000000000..1298a1a485 --- /dev/null +++ b/internal/libs/endpoint/matcher_test.go @@ -0,0 +1,48 @@ +package endpoint + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMatchers(t *testing.T) { + t.Parallel() + t.Run("dnsMatcherEmptyPattern", func(t *testing.T) { + d := dnsMatcher{pattern: ""} + assert.False(t, d.Match("foo")) + }) + t.Run("dnsMatcherBasic", func(t *testing.T) { + d := dnsMatcher{pattern: "foo"} + assert.True(t, d.Match("foo")) + }) + t.Run("dnsMatcherMatchingGlob", func(t *testing.T) { + d := dnsMatcher{pattern: "fo*o*"} + assert.True(t, d.Match("foweroasgdf")) + }) + t.Run("dnsMatcherNonMatchingGlob", func(t *testing.T) { + d := dnsMatcher{pattern: "fo*o*"} + assert.False(t, d.Match("bfoweroasgdf")) + }) + t.Run("cidrMatcherNilMatcher", func(t *testing.T) { + d := cidrMatcher{} + assert.False(t, d.Match("1.2.3.4")) + }) + t.Run("cidrMatcherIpv4Matcher", func(t *testing.T) { + _, net, err := net.ParseCIDR("1.2.3.4/24") + require.NoError(t, err) + d := cidrMatcher{ipNet: net} + assert.True(t, d.Match("1.2.3.4")) + assert.False(t, d.Match("1.2.4.4")) + assert.False(t, d.Match("260.234.19.3")) + }) + t.Run("cidrMatcherIpv6Matcher", func(t *testing.T) { + _, net, err := net.ParseCIDR("2001:1234::/32") + require.NoError(t, err) + d := cidrMatcher{ipNet: net} + assert.True(t, d.Match("2001:1234:3092::abcd:dead:beef:2423")) + assert.False(t, d.Match("2001:1244:3092::abcd:dead:beef:2423")) + }) +} diff --git a/internal/libs/endpoint/option.go b/internal/libs/endpoint/option.go new file mode 100644 index 0000000000..bf70a8624d --- /dev/null +++ b/internal/libs/endpoint/option.go @@ -0,0 +1,106 @@ +package endpoint + +import ( + "fmt" + "net" + "strings" +) + +// getOpts iterates the inbound Options and returns a struct +func getOpts(opt ...Option) (options, error) { + opts := getDefaultOptions() + for _, o := range opt { + if o == nil { + continue + } + if err := o(&opts); err != nil { + return options{}, err + } + } + return opts, nil +} + +// Option - how Options are passed as arguments +type Option func(*options) error + +// options = how options are represented +type options struct { + withIpAddrs []string + withDnsNames []string + withMatchers []matcher +} + +func getDefaultOptions() options { + return options{} +} + +// WithIpAddrs contains IP addresses to add into the endpoint possibilities. +// If an IP cannot be parsed, this function will error. +func WithIpAddrs(with []string) Option { + return func(o *options) error { + for _, addr := range with { + ip := net.ParseIP(addr) + if ip == nil { + return fmt.Errorf("input '%s' is not parseable as an ip address", addr) + } + o.withIpAddrs = append(o.withIpAddrs, addr) + } + return nil + } +} + +// WithDnsNames contains DNS names to add into the endpoint possibilities +func WithDnsNames(with []string) Option { + return func(o *options) error { + o.withDnsNames = with + return nil + } +} + +// WithPreferenceOrder contains the preference order specification. If one of +// the preferences cannot be parsed, this function will error. Internally it +// builds up a set of matchers. +func WithPreferenceOrder(with []string) Option { + return func(o *options) error { + for _, input := range with { + var m matcher + switch { + case strings.HasPrefix(input, "cidr:"): + // Make sure ParseCIDR won't choke on a bare address + cidr := strings.TrimPrefix(input, "cidr:") + if !strings.Contains(cidr, "/") { + // See if it seems like an IPv6 address vs. IPv4; colons are + // a good way to check this + if strings.Contains(cidr, ":") { + cidr = fmt.Sprintf("%s/128", cidr) + } else { + cidr = fmt.Sprintf("%s/32", cidr) + } + } + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("error parsing cidr %s: %w", cidr, err) + } + m = cidrMatcher{ + ipNet: ipNet, + } + + case strings.HasPrefix(input, "dns:"): + pattern := strings.TrimPrefix(input, "dns:") + if pattern == "" { + return fmt.Errorf("empty dns pattern provided") + } + m = dnsMatcher{ + pattern: pattern, + } + + default: + return fmt.Errorf("preference string %q is not supported", input) + } + if m != nil { + o.withMatchers = append(o.withMatchers, m) + } + } + return nil + } +} diff --git a/internal/libs/endpoint/options_test.go b/internal/libs/endpoint/options_test.go new file mode 100644 index 0000000000..1c87293dfb --- /dev/null +++ b/internal/libs/endpoint/options_test.go @@ -0,0 +1,79 @@ +package endpoint + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_GetOpts(t *testing.T) { + t.Parallel() + t.Run("WithNil", func(t *testing.T) { + _, err := getOpts(Option(nil)) + require.NoError(t, err) + }) + t.Run("WithDnsNames", func(t *testing.T) { + opts, err := getOpts(WithDnsNames([]string{"foo.bar", "fluebar"})) + require.NoError(t, err) + testOpts := getDefaultOptions() + testOpts.withDnsNames = []string{"foo.bar", "fluebar"} + assert.Equal(t, opts, testOpts) + }) + t.Run("WithIpAddrsBadIp", func(t *testing.T) { + _, err := getOpts(WithIpAddrs([]string{"foo.bar", "1.2.3.4"})) + require.Error(t, err) + }) + t.Run("WithIpAddrs", func(t *testing.T) { + opts, err := getOpts(WithIpAddrs([]string{"1.2.3.4", "5.6.7.8"})) + require.NoError(t, err) + testOpts := getDefaultOptions() + testOpts.withIpAddrs = []string{"1.2.3.4", "5.6.7.8"} + assert.Equal(t, opts, testOpts) + }) + t.Run("WithPreferenceOrderBadCidr", func(t *testing.T) { + _, err := getOpts(WithPreferenceOrder([]string{"cidr:15.3.25.6/33", "1.2.3.4"})) + require.Error(t, err) + }) + t.Run("WithPreferenceOrderBadDns", func(t *testing.T) { + _, err := getOpts(WithPreferenceOrder([]string{"dns:"})) + require.Error(t, err) + }) + t.Run("WithPreferenceOrderBadPref", func(t *testing.T) { + _, err := getOpts(WithPreferenceOrder([]string{"abc:15.3.25.6/33", "1.2.3.4"})) + require.Error(t, err) + }) + t.Run("WithPreferenceOrder", func(t *testing.T) { + require := require.New(t) + cidr1Str := "15.3.25.6/8" + _, net1, err := net.ParseCIDR(cidr1Str) + require.NoError(err) + dns1Str := "foo.bar" + cidr2Str := "2001::44" + _, net2, err := net.ParseCIDR(cidr2Str + "/128") + require.NoError(err) + cidr3Str := "1.2.3.4" + _, net3, err := net.ParseCIDR(cidr3Str + "/32") + require.NoError(err) + opts, err := getOpts(WithPreferenceOrder([]string{"cidr:" + cidr1Str, "dns:" + dns1Str, "cidr:" + cidr2Str, "cidr:" + cidr3Str})) + require.NoError(err) + testOpts := getDefaultOptions() + + testOpts.withMatchers = []matcher{ + cidrMatcher{ + ipNet: net1, + }, + dnsMatcher{ + pattern: dns1Str, + }, + cidrMatcher{ + ipNet: net2, + }, + cidrMatcher{ + ipNet: net3, + }, + } + assert.Equal(t, opts, testOpts) + }) +} diff --git a/internal/libs/endpoint/preferencer.go b/internal/libs/endpoint/preferencer.go new file mode 100644 index 0000000000..17be0893a3 --- /dev/null +++ b/internal/libs/endpoint/preferencer.go @@ -0,0 +1,76 @@ +package endpoint + +import ( + "context" + "math/rand" + "time" + + "github.com/hashicorp/boundary/internal/errors" +) + +type preferencer struct { + matchers []matcher +} + +// NewPreferencer builds up a preferencer with a set of preference options. This +// can then be used with Choose to select preferences from among a set of IP +// addresses and DNS names. +// +// Supported options: WithPreferenceOrder +func NewPreferencer(ctx context.Context, opt ...Option) (*preferencer, error) { + const op = "endpoint.NewPreferencer" + opts, err := getOpts(opt...) + if err != nil { + return nil, errors.Wrap(ctx, err, op) + } + pref := &preferencer{ + matchers: opts.withMatchers, + } + return pref, nil +} + +// Choose takes in IP addresses and/or DNS names and chooses an endpoint from +// among them, picking one at random if there are no preferences supplied. If +// preferences are specified but none match, the empty string is returned. +// However, if no IP addresses or DNS names are supplied, an error is returned. +// +// Supported options: WithIpAddrs, WithDnsNames +func (p *preferencer) Choose(ctx context.Context, opt ...Option) (string, error) { + const op = "endpoint.(preferencer).Choose" + opts, err := getOpts(opt...) + if err != nil { + return "", errors.Wrap(ctx, err, op) + } + if len(opts.withIpAddrs)+len(opts.withDnsNames) == 0 { + return "", errors.New(ctx, errors.InvalidParameter, op, "no ip addresses or dns names passed in") + } + + switch len(p.matchers) { + case 0: + // We have no matchers, so pick one at random + allAddrs := append(opts.withIpAddrs, opts.withDnsNames...) + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + return allAddrs[rng.Intn(len(allAddrs))], nil + + default: + for _, m := range p.matchers { + switch m.(type) { + case dnsMatcher: + for _, name := range opts.withDnsNames { + if m.Match(name) { + return name, nil + } + } + case cidrMatcher: + for _, addr := range opts.withIpAddrs { + if m.Match(addr) { + return addr, nil + } + } + } + } + // Nothing matched. Don't treat it as an error, let the calling function + // simply ignore the empty result. + return "", nil + } +} diff --git a/internal/libs/endpoint/preferencer_test.go b/internal/libs/endpoint/preferencer_test.go new file mode 100644 index 0000000000..917a75aa8f --- /dev/null +++ b/internal/libs/endpoint/preferencer_test.go @@ -0,0 +1,110 @@ +package endpoint + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPreferencer(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("badOption", func(t *testing.T) { + _, err := NewPreferencer(ctx, WithPreferenceOrder([]string{"bad:1.2.3.4"})) + assert.Error(t, err) + }) + t.Run("goodOption", func(t *testing.T) { + _, err := NewPreferencer(ctx, WithPreferenceOrder([]string{"cidr:1.2.3.4"})) + assert.NoError(t, err) + }) + t.Run("chooseBadOption", func(t *testing.T) { + p, err := NewPreferencer(ctx) + require.NoError(t, err) + _, err = p.Choose(ctx, WithIpAddrs([]string{"266.1.2.3"})) + assert.Error(t, err) + }) + t.Run("noAddresses", func(t *testing.T) { + p, err := NewPreferencer(ctx) + require.NoError(t, err) + _, err = p.Choose(ctx) + assert.Error(t, err) + }) + t.Run("preferenceOrder", func(t *testing.T) { + p, err := NewPreferencer(ctx, + WithPreferenceOrder([]string{ + "cidr:1.2.3.4/24", + "dns:*.example.com", + "cidr:5.6.7.8/8", + "dns:*.company.com", + })) + require.NoError(t, err) + + cases := []struct { + name string + withIpAddrs []string + withDnsNames []string + expectedEndpoint string + expectedErrorContains string + }{ + { + name: "first cidr", + withIpAddrs: []string{"5.6.7.8", "1.2.3.56"}, + withDnsNames: []string{"foo.bar", "bar.baz"}, + expectedEndpoint: "1.2.3.56", + }, + { + name: "first dns", + withIpAddrs: []string{"48.134.5.1", "1.2.7.56"}, + withDnsNames: []string{"bar.baz", "foo.example.com"}, + expectedEndpoint: "foo.example.com", + }, + { + name: "second cidr", + withIpAddrs: []string{"5.6.7.8", "1.2.7.56"}, + withDnsNames: []string{"foo.bar", "bar.baz"}, + expectedEndpoint: "5.6.7.8", + }, + { + name: "second dns", + withIpAddrs: []string{"48.134.5.1", "1.2.7.56"}, + withDnsNames: []string{"foo.bar.com", "bar.company.com"}, + expectedEndpoint: "bar.company.com", + }, + { + name: "no match", + withIpAddrs: []string{"48.134.5.1", "1.2.7.56"}, + withDnsNames: []string{"foo.bar.com", "bar.baz.com"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, err := p.Choose(ctx, WithIpAddrs(tc.withIpAddrs), WithDnsNames(tc.withDnsNames)) + if tc.expectedErrorContains != "" { + require.Error(t, err) + assert.Contains(t, tc.expectedErrorContains, err.Error()) + } + assert.Equal(t, tc.expectedEndpoint, out) + }) + } + }) + t.Run("noPrefRandom", func(t *testing.T) { + p, err := NewPreferencer(ctx) + require.NoError(t, err) + checkMap := map[string]int{} + for i := 0; i < 200; i++ { + out, err := p.Choose( + ctx, + WithIpAddrs([]string{"48.134.5.1", "1.2.7.56"}), + WithDnsNames([]string{"foo.bar.com", "bar.baz.com"}), + ) + require.NoError(t, err) + checkMap[out] = checkMap[out] + 1 + } + // Ensure that we've inserted all four keys + assert.Len(t, checkMap, 4) + }) +}