feat(host): Add host set information to static host (#1828)

* feat(host): Add host set information to static host

Updates LookupHost and ListHost API to include sets a static host is part of, as like with plugin hosts.

* test: updated TestList_Plugin and TestCrud to be more consistent under race conditions
pull/1839/head
Haotian 4 years ago committed by GitHub
parent 84f6cc698c
commit fa00a06bef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,24 @@
begin;
-- static_host_with_set_memberships is used for associating a static host instance with all its related host sets
-- in the set_ids column. Currently there are no size limits.
create view static_host_with_set_memberships as
select
h.public_id,
h.create_time,
h.update_time,
h.name,
h.description,
h.catalog_id,
h.address,
h.version,
-- the string_agg(..) column will be null if there are no associated value objects
string_agg(distinct hsm.set_id, '|') as set_ids
from
static_host h
left outer join static_host_set_member hsm on h.public_id = hsm.host_id
group by h.public_id;
comment on view static_host_with_set_memberships is
'static host with its associated host sets';
commit;

@ -22,4 +22,5 @@ type Host interface {
GetAddress() string
GetIpAddresses() []string
GetDnsNames() []string
GetSetIds() []string
}

@ -104,6 +104,11 @@ func (h *Host) oplog(op oplog.OpType) oplog.Metadata {
return metadata
}
// GetSetIds returns host set ids
func (h *Host) GetSetIds() []string {
return h.SetIds
}
// hostAgg is a view that aggregates the host's value objects in to
// string fields delimited with the aggregateDelimiter of "|"
type hostAgg struct {
@ -158,6 +163,7 @@ func (agg *hostAgg) TableName() string {
return "host_plugin_host_with_value_obj_and_set_memberships"
}
// GetPublicId returns the host public id as a string
func (agg *hostAgg) GetPublicId() string {
return agg.PublicId
}

@ -1,6 +1,10 @@
package static
import (
"sort"
"strings"
"github.com/hashicorp/boundary/internal/db/timestamp"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/host/static/store"
"github.com/hashicorp/boundary/internal/oplog"
@ -15,7 +19,8 @@ const (
// A Host contains a static address.
type Host struct {
*store.Host
tableName string `gorm:"-"`
SetIds []string `gorm:"-"`
tableName string `gorm:"-"`
}
// NewHost creates a new in memory Host for address assigned to catalogId.
@ -70,9 +75,18 @@ func allocHost() *Host {
func (h *Host) clone() *Host {
cp := proto.Clone(h.Host)
return &Host{
nh := &Host{
Host: cp.(*store.Host),
}
switch {
case h.SetIds == nil:
case len(h.SetIds) == 0:
nh.SetIds = make([]string, 0)
default:
nh.SetIds = make([]string, len(h.SetIds))
copy(nh.SetIds, h.SetIds)
}
return nh
}
func (h *Host) oplog(op oplog.OpType) oplog.Metadata {
@ -86,3 +100,55 @@ func (h *Host) oplog(op oplog.OpType) oplog.Metadata {
}
return metadata
}
// GetSetIds returns host set ids
func (h *Host) GetSetIds() []string {
return h.SetIds
}
type hostAgg struct {
PublicId string `gorm:"primary_key"`
CatalogId string
Name string
Description string
CreateTime *timestamp.Timestamp
UpdateTime *timestamp.Timestamp
Version uint32
Address string
SetIds string
}
func (agg *hostAgg) toHost() *Host {
h := allocHost()
h.PublicId = agg.PublicId
h.CatalogId = agg.CatalogId
h.Name = agg.Name
h.Description = agg.Description
h.CreateTime = agg.CreateTime
h.UpdateTime = agg.UpdateTime
h.Version = agg.Version
h.Address = agg.Address
h.SetIds = agg.getSetIds()
return h
}
// TableName returns the table name for gorm
func (agg *hostAgg) TableName() string {
return "static_host_with_set_memberships"
}
// GetPublicId returns the host public id as a string
func (agg *hostAgg) GetPublicId() string {
return agg.PublicId
}
// GetSetIds returns a list of all associated host sets to the host
func (agg *hostAgg) getSetIds() []string {
const aggregateDelimiter = "|"
var ids []string
if agg.SetIds != "" {
ids = strings.Split(agg.SetIds, aggregateDelimiter)
sort.Strings(ids)
}
return ids
}

@ -174,6 +174,13 @@ func (r *Repository) UpdateHost(ctx context.Context, scopeId string, h *Host, ve
if rowsUpdated > 1 {
return errors.New(ctx, errors.MultipleRecords, op, "more than 1 resource would have been updated")
}
ha := &hostAgg{
PublicId: h.PublicId,
}
if err := r.reader.LookupByPublicId(ctx, ha); err != nil {
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to lookup host after update"))
}
returnedHost.SetIds = ha.getSetIds()
return nil
},
)
@ -203,15 +210,16 @@ func (r *Repository) LookupHost(ctx context.Context, publicId string, opt ...Opt
if publicId == "" {
return nil, errors.New(ctx, errors.InvalidParameter, op, "no public id")
}
h := allocHost()
h.PublicId = publicId
if err := r.reader.LookupByPublicId(ctx, h); err != nil {
ha := &hostAgg{
PublicId: publicId,
}
if err := r.reader.LookupByPublicId(ctx, ha); err != nil {
if errors.IsNotFoundError(err) {
return nil, nil
}
return nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed for %s", publicId)))
}
return h, nil
return ha.toHost(), nil
}
// ListHosts returns a slice of Hosts for the catalogId.
@ -227,11 +235,16 @@ func (r *Repository) ListHosts(ctx context.Context, catalogId string, opt ...Opt
// non-zero signals an override of the default limit for the repo.
limit = opts.withLimit
}
var hosts []*Host
err := r.reader.SearchWhere(ctx, &hosts, "catalog_id = ?", []interface{}{catalogId}, db.WithLimit(limit))
var aggs []*hostAgg
err := r.reader.SearchWhere(ctx, &aggs, "catalog_id = ?", []interface{}{catalogId}, db.WithLimit(limit))
if err != nil {
return nil, errors.Wrap(ctx, err, op)
}
hosts := make([]*Host, 0, len(aggs))
for _, ha := range aggs {
hosts = append(hosts, ha.toHost())
}
return hosts, nil
}

@ -2,6 +2,7 @@ package static
import (
"context"
"sort"
"testing"
"time"
@ -9,6 +10,7 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/hashicorp/boundary/internal/db"
dbassert "github.com/hashicorp/boundary/internal/db/assert"
"github.com/hashicorp/boundary/internal/db/timestamp"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/host/static/store"
"github.com/hashicorp/boundary/internal/iam"
@ -604,6 +606,10 @@ func TestRepository_UpdateHost(t *testing.T) {
assert.NoError(err)
require.NotNil(orig)
set := TestSets(t, conn, catalog.GetPublicId(), 1)[0]
TestSetMembers(t, conn, set.PublicId, []*Host{orig})
wantSetIds := []string{set.PublicId}
if tt.chgFn != nil {
orig = tt.chgFn(orig)
}
@ -628,6 +634,7 @@ func TestRepository_UpdateHost(t *testing.T) {
dbassert.IsNull(got, "name")
return
}
assert.Equal(wantSetIds, got.SetIds)
assert.Equal(tt.want.Name, got.Name)
if tt.want.Description == "" {
dbassert.IsNull(got, "description")
@ -798,6 +805,72 @@ func TestRepository_LookupHost(t *testing.T) {
}
}
func TestRepository_LookupHost_HostSets(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
kms := kms.TestKms(t, conn, wrapper)
iamRepo := iam.TestRepo(t, conn, wrapper)
_, prj := iam.TestScopes(t, iamRepo)
catalog := TestCatalogs(t, conn, prj.PublicId, 1)[0]
hosts := TestHosts(t, conn, catalog.PublicId, 3)
hostA, hostB, hostC := hosts[0], hosts[1], hosts[2]
setB := TestSets(t, conn, catalog.PublicId, 1)[0]
setsC := TestSets(t, conn, catalog.PublicId, 5)
TestSetMembers(t, conn, setB.PublicId, []*Host{hostB})
hostB.SetIds = []string{setB.PublicId}
for _, s := range setsC {
hostC.SetIds = append(hostC.SetIds, s.PublicId)
TestSetMembers(t, conn, s.PublicId, []*Host{hostC})
}
sort.Slice(hostC.SetIds, func(i, j int) bool {
return hostC.SetIds[i] < hostC.SetIds[j]
})
tests := []struct {
name string
in string
want *Host
wantIsErr errors.Code
}{
{
name: "with-zero-hostsets",
in: hostA.PublicId,
want: hostA,
},
{
name: "with-one-hostset",
in: hostB.PublicId,
want: hostB,
},
{
name: "with-many-hostsets",
in: hostC.PublicId,
want: hostC,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
repo, err := NewRepository(rw, rw, kms)
assert.NoError(err)
require.NotNil(repo)
got, err := repo.LookupHost(context.Background(), tt.in)
assert.Empty(
cmp.Diff(
tt.want,
got,
cmpopts.IgnoreUnexported(Host{}, store.Host{}),
cmpopts.IgnoreTypes(&timestamp.Timestamp{}),
),
)
})
}
}
func TestRepository_ListHosts(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
@ -857,6 +930,80 @@ func TestRepository_ListHosts(t *testing.T) {
}
}
func TestRepository_ListHosts_HostSets(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
kms := kms.TestKms(t, conn, wrapper)
iamRepo := iam.TestRepo(t, conn, wrapper)
_, prj := iam.TestScopes(t, iamRepo)
// testing for full and empty hostset associations
catalogs := TestCatalogs(t, conn, prj.PublicId, 3)
catalogA, catalogB, catalogC := catalogs[0], catalogs[1], catalogs[2]
hostsA := TestHosts(t, conn, catalogA.PublicId, 3)
setA := TestSets(t, conn, catalogA.PublicId, 1)[0]
TestSetMembers(t, conn, setA.PublicId, hostsA)
for _, h := range hostsA {
h.SetIds = []string{setA.PublicId}
}
hostsB := TestHosts(t, conn, catalogB.PublicId, 3)
// testing for mixed hosts with individual hostsets and empty hostsets
hostsC := TestHosts(t, conn, catalogC.PublicId, 5)
hostsC0 := TestHosts(t, conn, catalogC.PublicId, 2)
setC := TestSets(t, conn, catalogC.PublicId, 5)
for i, h := range hostsC {
h.SetIds = []string{setC[i].PublicId}
TestSetMembers(t, conn, setC[i].PublicId, []*Host{hostsC[i]})
}
hostsC = append(hostsC, hostsC0...)
tests := []struct {
name string
in string
want []*Host
}{
{
name: "with-hostsets",
in: catalogA.PublicId,
want: hostsA,
},
{
name: "empty-hostsets",
in: catalogB.PublicId,
want: hostsB,
},
{
name: "mixed-hostsets",
in: catalogC.PublicId,
want: hostsC,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
repo, err := NewRepository(rw, rw, kms)
assert.NoError(err)
require.NotNil(repo)
got, err := repo.ListHosts(context.Background(), tt.in)
require.NoError(err)
assert.Empty(
cmp.Diff(
tt.want,
got,
cmpopts.IgnoreUnexported(Host{}, store.Host{}),
cmpopts.IgnoreTypes(&timestamp.Timestamp{}),
cmpopts.SortSlices(func(x, y *Host) bool {
return x.GetPublicId() < y.GetPublicId()
}),
),
)
})
}
}
func TestRepository_ListHosts_Limits(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)

@ -130,11 +130,7 @@ func (s Service) ListHosts(ctx context.Context, req *pbs.ListHostsRequest) (*pbs
if outputFields.Has(globals.AuthorizedActionsField) {
outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
}
switch h := item.(type) {
case *plugin.Host:
outputOpts = append(outputOpts, handlers.WithHostSetIds(h.SetIds))
}
outputOpts = append(outputOpts, handlers.WithHostSetIds(item.GetSetIds()))
item, err := toProto(ctx, item, outputOpts...)
if err != nil {
return nil, err
@ -180,11 +176,7 @@ func (s Service) GetHost(ctx context.Context, req *pbs.GetHostRequest) (*pbs.Get
idActions := idActionsTypeMap[host.SubtypeFromId(req.GetId())]
outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, h.GetPublicId(), idActions).Strings()))
}
switch h := h.(type) {
case *plugin.Host:
outputOpts = append(outputOpts, handlers.WithHostSetIds(h.SetIds))
}
outputOpts = append(outputOpts, handlers.WithHostSetIds(h.GetSetIds()))
item, err := toProto(ctx, h, outputOpts...)
if err != nil {
return nil, err
@ -265,6 +257,7 @@ func (s Service) UpdateHost(ctx context.Context, req *pbs.UpdateHostRequest) (*p
idActions := idActionsTypeMap[host.SubtypeFromId(req.GetId())]
outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, h.GetPublicId(), idActions).Strings()))
}
outputOpts = append(outputOpts, handlers.WithHostSetIds(h.GetSetIds()))
item, err := toProto(ctx, h, outputOpts...)
if err != nil {

@ -66,6 +66,8 @@ func TestGet_Static(t *testing.T) {
}
hc := static.TestCatalogs(t, conn, proj.GetPublicId(), 1)[0]
h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
s := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, s.GetPublicId(), []*static.Host{h})
pHost := &pb.Host{
HostCatalogId: hc.GetPublicId(),
@ -78,6 +80,7 @@ func TestGet_Static(t *testing.T) {
"address": structpb.NewStringValue(h.GetAddress()),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
}
cases := []struct {
@ -253,8 +256,11 @@ func TestList_Static(t *testing.T) {
hcs := static.TestCatalogs(t, conn, proj.GetPublicId(), 2)
hc, hcNoHosts := hcs[0], hcs[1]
hset := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
var wantHs []*pb.Host
for _, h := range static.TestHosts(t, conn, hc.GetPublicId(), 10) {
testHosts := static.TestHosts(t, conn, hc.GetPublicId(), 10)
static.TestSetMembers(t, conn, hset.GetPublicId(), testHosts)
for _, h := range testHosts {
wantHs = append(wantHs, &pb.Host{
Id: h.GetPublicId(),
HostCatalogId: h.GetCatalogId(),
@ -266,8 +272,12 @@ func TestList_Static(t *testing.T) {
"address": structpb.NewStringValue(h.GetAddress()),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{hset.GetPublicId()},
})
}
sort.Slice(wantHs, func(i int, j int) bool {
return wantHs[i].GetId() < wantHs[j].GetId()
})
cases := []struct {
name string
@ -793,11 +803,13 @@ func TestUpdate_Static(t *testing.T) {
require.NoError(t, err, "Couldn't create new static repo.")
hc := static.TestCatalogs(t, conn, proj.GetPublicId(), 1)[0]
s := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
h, err := static.NewHost(hc.GetPublicId(), static.WithName("default"), static.WithDescription("default"), static.WithAddress("defaultaddress"))
require.NoError(t, err)
h, err = repo.CreateHost(context.Background(), proj.GetPublicId(), h)
require.NoError(t, err)
static.TestSetMembers(t, conn, s.GetPublicId(), []*static.Host{h})
var version uint32 = 1
@ -847,6 +859,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},
@ -875,6 +888,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},
@ -945,6 +959,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},
@ -970,6 +985,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},
@ -997,6 +1013,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},
@ -1024,6 +1041,7 @@ func TestUpdate_Static(t *testing.T) {
"address": structpb.NewStringValue("defaultaddress"),
}},
AuthorizedActions: testAuthorizedActions[static.Subtype],
HostSetIds: []string{s.GetPublicId()},
},
},
},

@ -137,7 +137,14 @@ func TestList_Plugin(t *testing.T) {
ul, err = hClient.List(tc.Context(), hc.Item.Id)
require.NoError(err)
assert.ElementsMatch(comparableSetSlice(expected[:1]), comparableSetSlice(ul.Items))
assert.Empty(
cmp.Diff(
comparableSetSlice(expected[:1]),
comparableSetSlice(ul.Items),
cmpopts.IgnoreUnexported(hostsets.HostSet{}),
cmpopts.IgnoreFields(hostsets.HostSet{}, "Version", "UpdatedTime"),
),
)
for i := 1; i < 10; i++ {
hcr, err = hClient.Create(tc.Context(), hc.Item.Id, hostsets.WithName(expected[i].Name))
@ -196,18 +203,16 @@ func TestCrud(t *testing.T) {
require.NoError(err)
require.NotNil(hc)
retryableUpdate := func(c *hostsets.Client, hcId string, version uint32, opts ...hostsets.Option) (*hostsets.HostSetUpdateResult, bool) {
var retried bool
retryableUpdate := func(c *hostsets.Client, hcId string, version uint32, opts ...hostsets.Option) *hostsets.HostSetUpdateResult {
h, err := c.Update(tc.Context(), hcId, version, opts...)
if err != nil && strings.Contains(err.Error(), "set version mismatch") {
// Got a version mismatch, this happens because the sync set job runs in the background
// and can increment the version between operations in this test, try again
retried = true
h, err = c.Update(tc.Context(), hcId, version+1, opts...)
}
require.NoError(err)
assert.NotNil(h)
return h, retried
return h
}
hClient := hostsets.NewClient(client)
@ -246,7 +251,8 @@ func TestCrud(t *testing.T) {
require.NoError(err)
h, err = hClient.Create(tc.Context(), c.Item.Id, hostsets.WithName("foo"),
hostsets.WithAttributes(map[string]interface{}{"foo": "bar"}), hostsets.WithPreferredEndpoints([]string{"dns:test"}))
hostsets.WithAttributes(map[string]interface{}{"foo": "bar"}), hostsets.WithPreferredEndpoints([]string{"dns:test"}),
hostsets.WithSyncIntervalSeconds(-1))
require.NoError(err)
assert.Equal("foo", h.Item.Name)
assert.Equal(uint32(1), h.Item.Version)
@ -254,24 +260,20 @@ func TestCrud(t *testing.T) {
h, err = hClient.Read(tc.Context(), h.Item.Id)
require.NoError(err)
assert.Equal("foo", h.Item.Name)
assert.Equal(uint32(1), h.Item.Version)
// If the plugin set has synced after creation, its version will be 2; otherwise it will be 1.
assert.Contains([]uint32{1, 2}, h.Item.Version)
h, retried := retryableUpdate(hClient, h.Item.Id, h.Item.Version, hostsets.WithName("bar"),
h = retryableUpdate(hClient, h.Item.Id, h.Item.Version, hostsets.WithName("bar"),
hostsets.WithAttributes(map[string]interface{}{"foo": nil, "key": "val"}),
hostsets.WithPreferredEndpoints([]string{"dns:update"}))
require.NoError(err)
assert.Equal("bar", h.Item.Name)
switch retried {
case true:
assert.Equal(uint32(3), h.Item.Version)
default:
assert.Equal(uint32(2), h.Item.Version)
}
// If the plugin set has synced since creation, its version will be 3; otherwise it will be 2.
assert.Contains([]uint32{2, 3}, h.Item.Version)
assert.Equal(h.Item.Attributes, map[string]interface{}{"key": "val"})
assert.Equal(h.Item.PreferredEndpoints, []string{"dns:update"})
h, _ = retryableUpdate(hClient, h.Item.Id, h.Item.Version, hostsets.WithSyncIntervalSeconds(42))
h = retryableUpdate(hClient, h.Item.Id, h.Item.Version, hostsets.WithSyncIntervalSeconds(42))
require.NoError(err)
require.NotNil(h)
assert.Equal(int32(42), h.Item.SyncIntervalSeconds)

Loading…
Cancel
Save