aherman
/
boundary
mirror of https://github.com/hashicorp/boundary
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(e2e): Add rate limit test (#4135)

Browse Source
pull/4145/head
Michael Li 2 years ago committed by GitHub
parent ae2718291c
commit 9bf350d622
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 585 additions and 0 deletions
Show Stats Download Patch File Download Diff File

2
enos/enos-scenario-e2e-docker-base-plus.hcl
Unescape View File

@ -85,6 +85,8 @@ scenario "e2e_docker_base_plus" {
database_network = local.network_cluster
postgres_address = step.create_boundary_database.address
boundary_license = var.boundary_edition != "oss" ? step.read_license.license : ""
config_file = "boundary-config-rate-limit.hcl"
}
}

118
enos/modules/docker_boundary/boundary-config-rate-limit.hcl
Unescape View File

@ -0,0 +1,118 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
disable_mlock = true
controller {
name = "docker-controller"
database {
url = "env://BOUNDARY_POSTGRES_URL"
}
api_rate_limit {
resources = ["*"]
actions = ["*"]
per = "total"
unlimited = true
}
api_rate_limit {
resources = ["*"]
actions = ["*"]
per = "ip-address"
unlimited = true
}
api_rate_limit {
resources = ["*"]
actions = ["*"]
per = "auth-token"
unlimited = true
}
api_rate_limit {
resources = ["host"]
actions = ["read"]
per = "auth-token"
limit = 5
period = "5s"
}
api_rate_limit_max_quotas = 1
}
worker {
name = "boundary-collocated-worker"
description = "A worker that runs alongside the controller in the same process"
address = "boundary:9202"
tags {
type = ["${worker_type_tag}"]
}
}
listener "tcp" {
address = "boundary:9200"
purpose = "api"
tls_disable = true
}
listener "tcp" {
address = "boundary:9201"
purpose = "cluster"
tls_disable = true
}
listener "tcp" {
address = "boundary:9202"
purpose = "proxy"
tls_disable = true
}
listener "tcp" {
address = "boundary:9203"
purpose = "ops"
tls_disable = true
}
kms "aead" {
purpose = "root"
aead_type = "aes-gcm"
key = "sP1fnF5Xz85RrXyELHFeZg9Ad2qt4Z4bgNHVGtD6ung="
key_id = "global_root"
}
# This key_id needs to match the corresponding downstream worker's
# "worker-auth" kms
kms "aead" {
purpose = "worker-auth"
aead_type = "aes-gcm"
key = "OLFhJNbEb3umRjdhY15QKNEmNXokY1Iq"
key_id = "global_worker-auth"
}
kms "aead" {
purpose = "recovery"
aead_type = "aes-gcm"
key = "8fZBjCUfN0TzjEGLQldGY4+iE9AkOvCfjh7+p0GtRBQ="
key_id = "global_recovery"
}
events {
audit_enabled = true
observations_enabled = true
sysevents_enabled = true
sink "stderr" {
name = "all-events"
description = "All events sent to stderr"
event_types = ["*"]
format = "cloudevents-json"
deny_filters = [
"\"/data/request_info/method\" contains \"Status\"",
"\"/data/request_info/path\" contains \"/health\"",
]
}
}

8
testing/internal/e2e/boundary/boundary.go
Unescape View File

@ -36,3 +36,11 @@ type DbInitInfo struct {
type CliError struct {
Status int `json:"status_code"`
}
type HttpResponseBody struct {
Attributes HttpResponseBodyAttributes `json:"attributes"`
}
type HttpResponseBodyAttributes struct {
Token string `json:"token"`
}

33
testing/internal/e2e/boundary/http.go
Unescape View File

@ -0,0 +1,33 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package boundary
import (
"bytes"
"context"
"fmt"
"net/http"
"testing"
)
func AuthenticateHttp(t testing.TB, ctx context.Context, address, authMethodId, loginName, password string) (*http.Response, error) {
requestURL := fmt.Sprintf("%s/v1/auth-methods/%s:authenticate", address, authMethodId)
jsonBody := []byte(
fmt.Sprintf(`{"command":"login", "type":null, "attributes":{"login_name":"%s","password":"%s"}}`,
loginName,
password,
),
)
bodyReader := bytes.NewReader(jsonBody)
req, err := http.NewRequest(http.MethodPost, requestURL, bodyReader)
if err != nil {
return nil, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return res, err
}
return res, nil
}

6
testing/internal/e2e/helpers.go
Unescape View File

@ -11,6 +11,7 @@ import (
"os"
"os/exec"
"testing"
"time"
)
// CommandResult captures the output from running an external command
@ -19,6 +20,7 @@ type CommandResult struct {
Stderr []byte
ExitCode int
Err error
Duration time.Duration
}
// Option is a func that sets optional attributes for a call. This does not need
@ -77,7 +79,10 @@ func RunCommand(ctx context.Context, command string, opt ...Option) *CommandResu
cmd.Stdout = &outbuf
cmd.Stderr = &errbuf
startTime := time.Now()
err := cmd.Run()
endTime := time.Now()
duration := endTime.Sub(startTime)
var ee *exec.ExitError
var exitCode int
@ -90,6 +95,7 @@ func RunCommand(ctx context.Context, command string, opt ...Option) *CommandResu
Stderr: errbuf.Bytes(),
ExitCode: exitCode,
Err: err,
Duration: duration,
}
}

418
testing/internal/e2e/tests/base_plus/rate_limit_test.go
Unescape View File

@ -0,0 +1,418 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package base_plus_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"testing"
"time"
"github.com/hashicorp/boundary/testing/internal/e2e"
"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
"github.com/stretchr/testify/require"
)
// TestHttpRateLimit tests rate limiting when making HTTP API requests. When a
// request is rate limited, a HTTP 429 response is received. If a request is
// limited due to exceeding the quota limit, a HTTP 503 response is received.
// This test assumes that `api_rate_limit_max_quotas` in the config is set to 1
func TestHttpRateLimit(t *testing.T) {
e2e.MaybeSkipTest(t)
c, err := loadTestConfig()
require.NoError(t, err)
bc, err := boundary.LoadConfig()
require.NoError(t, err)
ctx := context.Background()
boundary.AuthenticateAdminCli(t, ctx)
newOrgId := boundary.CreateNewOrgCli(t, ctx)
t.Cleanup(func() {
ctx := context.Background()
boundary.AuthenticateAdminCli(t, ctx)
output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgId))
require.NoError(t, output.Err, string(output.Stderr))
})
newProjectId := boundary.CreateNewProjectCli(t, ctx, newOrgId)
newHostCatalogId := boundary.CreateNewHostCatalogCli(t, ctx, newProjectId)
newHostId := boundary.CreateNewHostCli(t, ctx, newHostCatalogId, c.TargetAddress)
// Authenticate over HTTP
res, err := boundary.AuthenticateHttp(t, ctx, bc.Address, bc.AuthMethodId, bc.AdminLoginName, bc.AdminLoginPassword)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
body, err := io.ReadAll(res.Body)
require.NoError(t, err)
var r boundary.HttpResponseBody
err = json.Unmarshal(body, &r)
require.NoError(t, err)
tokenAdmin := r.Attributes.Token
res.Body.Close()
// Make initial API request
t.Log("Sending API requests until quota is hit...")
requestURL := fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
body, err = io.ReadAll(res.Body)
require.NoError(t, err)
res.Body.Close()
require.NotEmpty(t, body)
require.Contains(t, string(body), newHostId)
// Check that the limit from the policy matches the actual limit
rateLimitPolicyHeader := res.Header.Get("Ratelimit-Policy")
require.NotEmpty(t, rateLimitPolicyHeader)
policyLimit, policyPeriod, err := getRateLimitPolicyStat(res.Header.Get("Ratelimit-Policy"), "auth-token")
require.NoError(t, err)
t.Log(rateLimitPolicyHeader)
rateLimitHeader := res.Header.Get("Ratelimit")
require.NotEmpty(t, rateLimitHeader)
t.Log(rateLimitHeader)
limit, err := getRateLimitStat(rateLimitHeader, "limit")
require.NoError(t, err)
require.Equal(t, policyLimit, limit)
// Make API requests until quota is hit
quota, err := getRateLimitStat(rateLimitHeader, "remaining")
require.NoError(t, err)
for quota > 0 {
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
rateLimitHeader := res.Header.Get("Ratelimit")
require.NotEmpty(t, rateLimitHeader)
t.Log(rateLimitHeader)
remaining, err := getRateLimitStat(rateLimitHeader, "remaining")
require.NoError(t, err)
require.Equal(t, quota-1, remaining, "Remaining quota did not decrease after sending API request")
quota = remaining
}
// Do another request after the quota is exhausted
// Verify that the request is not successful (HTTP 429: Too Many Requests)
t.Log("Checking that next API request is rate limited...")
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusTooManyRequests, res.StatusCode)
body, err = io.ReadAll(res.Body)
require.NoError(t, err)
res.Body.Close()
require.Empty(t, body)
// Wait for "Retry-After" time
retryAfterHeader := res.Header.Get("Retry-After")
require.NotEmpty(t, retryAfterHeader)
retryAfter, err := strconv.Atoi(retryAfterHeader)
require.NoError(t, err)
t.Logf("Waiting for %d seconds to retry API request...", retryAfter)
time.Sleep(time.Duration(retryAfter) * time.Second)
// Do another request. Verify that request is successful
t.Logf("Retrying...")
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
body, err = io.ReadAll(res.Body)
require.NoError(t, err)
res.Body.Close()
require.NotEmpty(t, body)
require.Contains(t, string(body), newHostId)
t.Log("Successfully sent request after waiting")
// Create a user
acctName := "e2e-account"
newAccountId, acctPassword := boundary.CreateNewAccountCli(t, ctx, bc.AuthMethodId, acctName)
t.Cleanup(func() {
boundary.AuthenticateAdminCli(t, context.Background())
output := e2e.RunCommand(ctx, "boundary",
e2e.WithArgs("accounts", "delete", "-id", newAccountId),
)
require.NoError(t, output.Err, string(output.Stderr))
})
newUserId := boundary.CreateNewUserCli(t, ctx, "global")
t.Cleanup(func() {
boundary.AuthenticateAdminCli(t, context.Background())
output := e2e.RunCommand(ctx, "boundary",
e2e.WithArgs("users", "delete", "-id", newUserId),
)
require.NoError(t, output.Err, string(output.Stderr))
})
boundary.SetAccountToUserCli(t, ctx, newUserId, newAccountId)
newRoleId := boundary.CreateNewRoleCli(t, ctx, newProjectId)
boundary.AddGrantToRoleCli(t, ctx, newRoleId, "ids=*;type=*;actions=*")
boundary.AddPrincipalToRoleCli(t, ctx, newRoleId, newUserId)
// Get auth token for second user
res, err = boundary.AuthenticateHttp(t, ctx, bc.Address, bc.AuthMethodId, acctName, acctPassword)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
body, err = io.ReadAll(res.Body)
require.NoError(t, err)
err = json.Unmarshal(body, &r)
require.NoError(t, err)
tokenUser := r.Attributes.Token
res.Body.Close()
// Read target until quota is hit again using the first user
t.Log("Sending API requests until quota is hit...")
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
rateLimitHeader = res.Header.Get("Ratelimit")
require.NotEmpty(t, rateLimitHeader)
t.Log(rateLimitHeader)
quota, err = getRateLimitStat(rateLimitHeader, "remaining")
require.NoError(t, err)
for quota > 0 {
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
rateLimitHeader := res.Header.Get("Ratelimit")
require.NotEmpty(t, rateLimitHeader)
t.Log(rateLimitHeader)
remaining, err := getRateLimitStat(rateLimitHeader, "remaining")
require.NoError(t, err)
require.Equal(t, quota-1, remaining, "Remaining quota did not decrease after sending API request")
quota = remaining
}
// Confirm that a request from the second user results in a HTTP 503 due to
// exceeding the quota limit
t.Log("Checking that next API request is rejected...")
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenUser))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusServiceUnavailable, res.StatusCode)
body, err = io.ReadAll(res.Body)
require.NoError(t, err)
res.Body.Close()
require.Empty(t, body)
// Wait for "Retry-After" time
// Note: Not using the Retry-After time from an HTTP 503 response for now
//
// retryAfterHeader = res.Header.Get("Retry-After")
// require.NotEmpty(t, retryAfterHeader)
// retryAfter, err = strconv.Atoi(retryAfterHeader)
// require.NoError(t, err)
t.Logf("Waiting for %d seconds to retry API request...", policyPeriod)
time.Sleep(time.Duration(policyPeriod) * time.Second)
// Do another request. Verify that request is successful
t.Log("Retrying...")
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
require.NoError(t, err)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenUser))
res, err = http.DefaultClient.Do(req)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)
t.Log("Successfully sent request after waiting")
time.Sleep(time.Duration(policyPeriod) * time.Second)
t.Logf("Waiting for %d seconds for quote to expire...", policyPeriod)
}
// TestCliRateLimit tests rate limiting when using the boundary cli. When a
// request is rate limited, a 429 response is received and if a request is quota
// limited, a 503 response is received. If the BOUNDARY_MAX_RETRIES environment
// variable it set, it will instead auto-retry, automatically trying again after
// Retry-After seconds (from the response header)
func TestCliRateLimit(t *testing.T) {
e2e.MaybeSkipTest(t)
c, err := loadTestConfig()
require.NoError(t, err)
bc, err := boundary.LoadConfig()
require.NoError(t, err)
ctx := context.Background()
boundary.AuthenticateAdminCli(t, ctx)
newOrgId := boundary.CreateNewOrgCli(t, ctx)
t.Cleanup(func() {
ctx := context.Background()
boundary.AuthenticateAdminCli(t, ctx)
output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgId))
require.NoError(t, output.Err, string(output.Stderr))
})
newProjectId := boundary.CreateNewProjectCli(t, ctx, newOrgId)
newHostCatalogId := boundary.CreateNewHostCatalogCli(t, ctx, newProjectId)
newHostId := boundary.CreateNewHostCli(t, ctx, newHostCatalogId, c.TargetAddress)
// Create a user
acctName := "e2e-account"
newAccountId, acctPassword := boundary.CreateNewAccountCli(t, ctx, bc.AuthMethodId, acctName)
t.Cleanup(func() {
boundary.AuthenticateAdminCli(t, context.Background())
output := e2e.RunCommand(ctx, "boundary",
e2e.WithArgs("accounts", "delete", "-id", newAccountId),
)
require.NoError(t, output.Err, string(output.Stderr))
})
newUserId := boundary.CreateNewUserCli(t, ctx, "global")
t.Cleanup(func() {
boundary.AuthenticateAdminCli(t, context.Background())
output := e2e.RunCommand(ctx, "boundary",
e2e.WithArgs("users", "delete", "-id", newUserId),
)
require.NoError(t, output.Err, string(output.Stderr))
})
boundary.SetAccountToUserCli(t, ctx, newUserId, newAccountId)
newRoleId := boundary.CreateNewRoleCli(t, ctx, newProjectId)
boundary.AddGrantToRoleCli(t, ctx, newRoleId, "ids=*;type=*;actions=*")
boundary.AddPrincipalToRoleCli(t, ctx, newRoleId, newUserId)
// Run tests until rate limit is hit. Expect to see a HTTP 429 when rate limited
t.Log("Sending multiple CLI requests to hit rate limit...")
var output *e2e.CommandResult
for i := 0; i < 20; i++ {
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
t.Log(output.Duration)
if output.Err != nil {
break
}
require.NoError(t, output.Err, string(output.Stderr))
require.Equal(t, 0, output.ExitCode)
}
require.Error(t, output.Err, string(output.Stderr))
require.Equal(t, 1, output.ExitCode)
require.Contains(t, string(output.Stderr), strconv.Itoa(http.StatusTooManyRequests))
t.Log("Successfully observed a HTTP 429 response")
// Log in as a second user and confirm you get a HTTP 503 response
t.Log("Logging in as another user...")
boundary.AuthenticateCli(t, ctx, bc.AuthMethodId, acctName, acctPassword)
for i := 0; i < 20; i++ {
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
t.Log(output.Duration)
if output.Err != nil {
break
}
require.NoError(t, output.Err, string(output.Stderr))
require.Equal(t, 0, output.ExitCode)
}
require.Error(t, output.Err, string(output.Stderr))
require.Equal(t, 1, output.ExitCode)
require.Contains(t, string(output.Stderr), strconv.Itoa(http.StatusServiceUnavailable))
t.Log("Successfully observed a HTTP 503 response")
// Setting this environment variable sets CLI to use an auto-retry when rate
// limited
boundary.AuthenticateAdminCli(t, ctx)
t.Log("Setting BOUNDARY_MAX_RETRIES environment variable...")
os.Setenv("BOUNDARY_MAX_RETRIES", "2")
t.Cleanup(func() {
os.Unsetenv("BOUNDARY_MAX_RETRIES")
})
// Run tests until rate limit is hit. Expect to see the CLI auto-retry (the
// command will take longer to return)
t.Log("Sending multiple CLI requests to hit rate limit...")
threshold := time.Duration(3 * time.Second)
for i := 0; i < 20; i++ {
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
t.Log(output.Duration)
if output.Duration >= threshold {
break
}
}
require.NoError(t, output.Err, string(output.Stderr))
require.Equal(t, 0, output.ExitCode)
require.GreaterOrEqual(t, output.Duration, threshold, "CLI did not auto-retry request")
t.Logf("Successfully auto-retried CLI request")
}
func getRateLimitStat(rateLimitHeader, stat string) (int, error) {
ss := strings.Split(rateLimitHeader, ", ")
for _, s := range ss {
if strings.Contains(s, stat) {
parts := strings.Split(s, "=")
if len(parts) != 2 {
return 0, errors.New(fmt.Sprintf("Expected length of 2: VALUE: %s", parts))
}
count, err := strconv.Atoi(parts[1])
if err != nil {
return 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %s", parts[1]))
}
return count, nil
}
}
return 0, errors.New(fmt.Sprintf("Could not parse header, STAT: %s, HEADER: %s", stat, rateLimitHeader))
}
func getRateLimitPolicyStat(rateLimitPolicyHeader, stat string) (limit int, period int, err error) {
ss := strings.Split(rateLimitPolicyHeader, ", ")
for _, s := range ss {
if strings.Contains(s, stat) {
parts := strings.Split(s, ";")
if len(parts) != 3 {
return 0, 0, errors.New(fmt.Sprintf("Expected length of 3: VALUE: %s", parts))
}
limit, err := strconv.Atoi(parts[0])
if err != nil {
return 0, 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %s", parts[0]))
}
policyParts := strings.Split(parts[1], "=")
if len(policyParts) != 2 {
return 0, 0, errors.New(fmt.Sprintf("Expected length of 2: VALUE: %s", policyParts))
}
period, err := strconv.Atoi(policyParts[1])
if err != nil {
return 0, 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %d", period))
}
return limit, period, nil
}
}
return 0, 0, errors.New(fmt.Sprintf("Could not parse header, STAT: %s, HEADER: %s", stat, rateLimitPolicyHeader))
}
Write Preview
Loading…
Cancel
Save