mirror of https://github.com/hashicorp/boundary
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
446 lines
16 KiB
446 lines
16 KiB
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package base_plus_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/boundary/testing/internal/e2e"
|
|
"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestHttpRateLimit tests rate limiting when making HTTP API requests. When a
|
|
// request is rate limited, a HTTP 429 response is received. If a request is
|
|
// limited due to exceeding the quota limit, a HTTP 503 response is received.
|
|
// This test assumes that `api_rate_limit_max_quotas` in the config is set to 1
|
|
func TestHttpRateLimit(t *testing.T) {
|
|
e2e.MaybeSkipTest(t)
|
|
c, err := loadTestConfig()
|
|
require.NoError(t, err)
|
|
bc, err := boundary.LoadConfig()
|
|
require.NoError(t, err)
|
|
|
|
ctx := context.Background()
|
|
boundary.AuthenticateAdminCli(t, ctx)
|
|
newOrgId := boundary.CreateNewOrgCli(t, ctx)
|
|
t.Cleanup(func() {
|
|
ctx := context.Background()
|
|
boundary.AuthenticateAdminCli(t, ctx)
|
|
output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgId))
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
newProjectId := boundary.CreateNewProjectCli(t, ctx, newOrgId)
|
|
newHostCatalogId := boundary.CreateNewHostCatalogCli(t, ctx, newProjectId)
|
|
newHostId := boundary.CreateNewHostCli(t, ctx, newHostCatalogId, c.TargetAddress)
|
|
|
|
// Authenticate over HTTP
|
|
res, err := boundary.AuthenticateHttp(t, ctx, bc.Address, bc.AuthMethodId, bc.AdminLoginName, bc.AdminLoginPassword)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
body, err := io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
var r boundary.HttpResponseBody
|
|
err = json.Unmarshal(body, &r)
|
|
require.NoError(t, err)
|
|
tokenAdmin := r.Attributes.Token
|
|
res.Body.Close()
|
|
|
|
// Make initial API request
|
|
t.Log("Sending API requests until quota is hit...")
|
|
requestURL := fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
|
|
body, err = io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
res.Body.Close()
|
|
require.NotEmpty(t, body)
|
|
require.Contains(t, string(body), newHostId)
|
|
|
|
// Check that the limit from the policy matches the actual limit
|
|
rateLimitPolicyHeader := res.Header.Get("Ratelimit-Policy")
|
|
require.NotEmpty(t, rateLimitPolicyHeader)
|
|
policyLimit, policyPeriod, err := getRateLimitPolicyStat(res.Header.Get("Ratelimit-Policy"), "auth-token")
|
|
require.NoError(t, err)
|
|
t.Log(rateLimitPolicyHeader)
|
|
|
|
rateLimitHeader := res.Header.Get("Ratelimit")
|
|
require.NotEmpty(t, rateLimitHeader)
|
|
t.Log(rateLimitHeader)
|
|
limit, err := getRateLimitStat(rateLimitHeader, "limit")
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, policyLimit, limit)
|
|
|
|
// Make API requests until quota is hit
|
|
quota, err := getRateLimitStat(rateLimitHeader, "remaining")
|
|
require.NoError(t, err)
|
|
for quota > 0 {
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
|
|
rateLimitHeader := res.Header.Get("Ratelimit")
|
|
require.NotEmpty(t, rateLimitHeader)
|
|
t.Log(rateLimitHeader)
|
|
remaining, err := getRateLimitStat(rateLimitHeader, "remaining")
|
|
require.NoError(t, err)
|
|
require.Equal(t, quota-1, remaining, "Remaining quota did not decrease after sending API request")
|
|
|
|
quota = remaining
|
|
}
|
|
|
|
// Do another request after the quota is exhausted
|
|
// Verify that the request is not successful (HTTP 429: Too Many Requests)
|
|
t.Log("Checking that next API request is rate limited...")
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusTooManyRequests, res.StatusCode)
|
|
|
|
body, err = io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
res.Body.Close()
|
|
require.Empty(t, body)
|
|
|
|
// Wait for "Retry-After" time
|
|
retryAfterHeader := res.Header.Get("Retry-After")
|
|
require.NotEmpty(t, retryAfterHeader)
|
|
retryAfter, err := strconv.Atoi(retryAfterHeader)
|
|
require.NoError(t, err)
|
|
t.Logf("Waiting for %d seconds to retry API request...", retryAfter)
|
|
time.Sleep(time.Duration(retryAfter) * time.Second)
|
|
|
|
// Do another request. Verify that request is successful
|
|
t.Logf("Retrying...")
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
|
|
body, err = io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
res.Body.Close()
|
|
require.NotEmpty(t, body)
|
|
require.Contains(t, string(body), newHostId)
|
|
|
|
t.Log("Successfully sent request after waiting")
|
|
|
|
// Create a user
|
|
acctName := "e2e-account"
|
|
newAccountId, acctPassword := boundary.CreateNewAccountCli(t, ctx, bc.AuthMethodId, acctName)
|
|
t.Cleanup(func() {
|
|
boundary.AuthenticateAdminCli(t, context.Background())
|
|
output := e2e.RunCommand(ctx, "boundary",
|
|
e2e.WithArgs("accounts", "delete", "-id", newAccountId),
|
|
)
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
newUserId := boundary.CreateNewUserCli(t, ctx, "global")
|
|
t.Cleanup(func() {
|
|
boundary.AuthenticateAdminCli(t, context.Background())
|
|
output := e2e.RunCommand(ctx, "boundary",
|
|
e2e.WithArgs("users", "delete", "-id", newUserId),
|
|
)
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
boundary.SetAccountToUserCli(t, ctx, newUserId, newAccountId)
|
|
newRoleId, err := boundary.CreateRoleCli(t, ctx, newProjectId)
|
|
require.NoError(t, err)
|
|
boundary.AddGrantToRoleCli(t, ctx, newRoleId, "ids=*;type=*;actions=*")
|
|
boundary.AddPrincipalToRoleCli(t, ctx, newRoleId, newUserId)
|
|
|
|
// Get auth token for second user
|
|
res, err = boundary.AuthenticateHttp(t, ctx, bc.Address, bc.AuthMethodId, acctName, acctPassword)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
body, err = io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
err = json.Unmarshal(body, &r)
|
|
require.NoError(t, err)
|
|
tokenUser := r.Attributes.Token
|
|
res.Body.Close()
|
|
|
|
// Make request until quota is hit again using the first user
|
|
t.Log("Sending API requests until quota is hit...")
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
rateLimitHeader = res.Header.Get("Ratelimit")
|
|
require.NotEmpty(t, rateLimitHeader)
|
|
t.Log(rateLimitHeader)
|
|
quota, err = getRateLimitStat(rateLimitHeader, "remaining")
|
|
require.NoError(t, err)
|
|
for quota > 0 {
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
|
|
rateLimitHeader := res.Header.Get("Ratelimit")
|
|
require.NotEmpty(t, rateLimitHeader)
|
|
t.Log(rateLimitHeader)
|
|
remaining, err := getRateLimitStat(rateLimitHeader, "remaining")
|
|
require.NoError(t, err)
|
|
require.Equal(t, quota-1, remaining, "Remaining quota did not decrease after sending API request")
|
|
|
|
quota = remaining
|
|
}
|
|
|
|
// Confirm that a request from the second user results in a HTTP 503 due to
|
|
// exceeding the quota limit
|
|
t.Log("Checking that next API request is rejected...")
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenUser))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusServiceUnavailable, res.StatusCode)
|
|
|
|
body, err = io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
res.Body.Close()
|
|
require.Empty(t, body)
|
|
|
|
// Wait for "Retry-After" time
|
|
// Note: Not using the Retry-After time from an HTTP 503 response for now
|
|
//
|
|
// retryAfterHeader = res.Header.Get("Retry-After")
|
|
// require.NotEmpty(t, retryAfterHeader)
|
|
// retryAfter, err = strconv.Atoi(retryAfterHeader)
|
|
// require.NoError(t, err)
|
|
t.Logf("Waiting for %d seconds to retry API request...", policyPeriod)
|
|
time.Sleep(time.Duration(policyPeriod) * time.Second)
|
|
|
|
// Do another request. Verify that request is successful
|
|
t.Log("Retrying...")
|
|
requestURL = fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err = http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenUser))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
t.Log("Successfully sent request after waiting")
|
|
|
|
time.Sleep(time.Duration(policyPeriod) * time.Second)
|
|
t.Logf("Waiting for %d seconds for quote to expire...", policyPeriod)
|
|
}
|
|
|
|
// TestCliRateLimit tests rate limiting when using the boundary cli. When a
|
|
// request is rate limited, a 429 response is received and if a request is quota
|
|
// limited, a 503 response is received. If the BOUNDARY_MAX_RETRIES environment
|
|
// variable it set, it will instead auto-retry, automatically trying again after
|
|
// Retry-After seconds (from the response header)
|
|
func TestCliRateLimit(t *testing.T) {
|
|
e2e.MaybeSkipTest(t)
|
|
c, err := loadTestConfig()
|
|
require.NoError(t, err)
|
|
|
|
bc, err := boundary.LoadConfig()
|
|
require.NoError(t, err)
|
|
|
|
ctx := context.Background()
|
|
|
|
boundary.AuthenticateAdminCli(t, ctx)
|
|
newOrgId := boundary.CreateNewOrgCli(t, ctx)
|
|
t.Cleanup(func() {
|
|
ctx := context.Background()
|
|
boundary.AuthenticateAdminCli(t, ctx)
|
|
output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgId))
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
newProjectId := boundary.CreateNewProjectCli(t, ctx, newOrgId)
|
|
newHostCatalogId := boundary.CreateNewHostCatalogCli(t, ctx, newProjectId)
|
|
newHostId := boundary.CreateNewHostCli(t, ctx, newHostCatalogId, c.TargetAddress)
|
|
|
|
// Create a user
|
|
acctName := "e2e-account"
|
|
newAccountId, acctPassword := boundary.CreateNewAccountCli(t, ctx, bc.AuthMethodId, acctName)
|
|
t.Cleanup(func() {
|
|
boundary.AuthenticateAdminCli(t, context.Background())
|
|
output := e2e.RunCommand(ctx, "boundary",
|
|
e2e.WithArgs("accounts", "delete", "-id", newAccountId),
|
|
)
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
newUserId := boundary.CreateNewUserCli(t, ctx, "global")
|
|
t.Cleanup(func() {
|
|
boundary.AuthenticateAdminCli(t, context.Background())
|
|
output := e2e.RunCommand(ctx, "boundary",
|
|
e2e.WithArgs("users", "delete", "-id", newUserId),
|
|
)
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
})
|
|
boundary.SetAccountToUserCli(t, ctx, newUserId, newAccountId)
|
|
newRoleId, err := boundary.CreateRoleCli(t, ctx, newProjectId)
|
|
require.NoError(t, err)
|
|
boundary.AddGrantToRoleCli(t, ctx, newRoleId, "ids=*;type=*;actions=*")
|
|
boundary.AddPrincipalToRoleCli(t, ctx, newRoleId, newUserId)
|
|
|
|
// Authenticate over HTTP
|
|
res, err := boundary.AuthenticateHttp(t, ctx, bc.Address, bc.AuthMethodId, bc.AdminLoginName, bc.AdminLoginPassword)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
body, err := io.ReadAll(res.Body)
|
|
require.NoError(t, err)
|
|
var r boundary.HttpResponseBody
|
|
err = json.Unmarshal(body, &r)
|
|
require.NoError(t, err)
|
|
tokenAdmin := r.Attributes.Token
|
|
res.Body.Close()
|
|
|
|
// Make initial API request
|
|
t.Log("Getting rate limit info...")
|
|
requestURL := fmt.Sprintf("%s/v1/hosts/%s", bc.Address, newHostId)
|
|
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokenAdmin))
|
|
res, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, http.StatusOK, res.StatusCode)
|
|
|
|
rateLimitPolicyHeader := res.Header.Get("Ratelimit-Policy")
|
|
require.NotEmpty(t, rateLimitPolicyHeader)
|
|
policyLimit, policyPeriod, err := getRateLimitPolicyStat(res.Header.Get("Ratelimit-Policy"), "auth-token")
|
|
require.NoError(t, err)
|
|
// Wait for ratelimit to reset
|
|
time.Sleep(time.Duration(policyPeriod) * time.Second)
|
|
|
|
// Run tests until rate limit is hit. Expect to see a HTTP 429 when rate limited
|
|
t.Log("Sending multiple CLI requests to hit rate limit...")
|
|
var output *e2e.CommandResult
|
|
for i := 0; i <= policyLimit; i++ {
|
|
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
|
|
t.Log(output.Duration)
|
|
if output.Err != nil {
|
|
break
|
|
}
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
require.Equal(t, 0, output.ExitCode)
|
|
}
|
|
require.Error(t, output.Err, string(output.Stderr))
|
|
require.Equal(t, 1, output.ExitCode)
|
|
require.Contains(t, string(output.Stderr), strconv.Itoa(http.StatusTooManyRequests))
|
|
t.Log("Successfully observed a HTTP 429 response")
|
|
|
|
// Log in as a second user and confirm you get a HTTP 503 response
|
|
t.Log("Logging in as another user...")
|
|
boundary.AuthenticateCli(t, ctx, bc.AuthMethodId, acctName, acctPassword)
|
|
for i := 0; i <= policyLimit; i++ {
|
|
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
|
|
t.Log(output.Duration)
|
|
if output.Err != nil {
|
|
break
|
|
}
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
require.Equal(t, 0, output.ExitCode)
|
|
}
|
|
require.Error(t, output.Err, string(output.Stderr))
|
|
require.Equal(t, 1, output.ExitCode)
|
|
require.Contains(t, string(output.Stderr), strconv.Itoa(http.StatusServiceUnavailable))
|
|
t.Log("Successfully observed a HTTP 503 response")
|
|
time.Sleep(time.Duration(policyPeriod) * time.Second)
|
|
|
|
// Setting this environment variable sets CLI to use an auto-retry when rate
|
|
// limited
|
|
boundary.AuthenticateAdminCli(t, ctx)
|
|
t.Log("Setting BOUNDARY_MAX_RETRIES environment variable...")
|
|
os.Setenv("BOUNDARY_MAX_RETRIES", "2")
|
|
t.Cleanup(func() {
|
|
os.Unsetenv("BOUNDARY_MAX_RETRIES")
|
|
})
|
|
|
|
// Run tests until rate limit is hit. Expect to see the CLI auto-retry (the
|
|
// command will take longer to return)
|
|
t.Log("Sending multiple CLI requests to hit rate limit...")
|
|
for i := 0; i <= policyLimit; i++ {
|
|
output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("hosts", "read", "-id", newHostId))
|
|
t.Log(output.Duration)
|
|
require.NoError(t, output.Err, string(output.Stderr))
|
|
require.Equal(t, 0, output.ExitCode)
|
|
}
|
|
t.Logf("Successfully auto-retried CLI request")
|
|
}
|
|
|
|
func getRateLimitStat(rateLimitHeader, stat string) (int, error) {
|
|
ss := strings.Split(rateLimitHeader, ", ")
|
|
for _, s := range ss {
|
|
if strings.Contains(s, stat) {
|
|
parts := strings.Split(s, "=")
|
|
if len(parts) != 2 {
|
|
return 0, errors.New(fmt.Sprintf("Expected length of 2: VALUE: %s", parts))
|
|
}
|
|
count, err := strconv.Atoi(parts[1])
|
|
if err != nil {
|
|
return 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %s", parts[1]))
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
}
|
|
|
|
return 0, errors.New(fmt.Sprintf("Could not parse header, STAT: %s, HEADER: %s", stat, rateLimitHeader))
|
|
}
|
|
|
|
func getRateLimitPolicyStat(rateLimitPolicyHeader, stat string) (limit int, period int, err error) {
|
|
ss := strings.Split(rateLimitPolicyHeader, ", ")
|
|
for _, s := range ss {
|
|
if strings.Contains(s, stat) {
|
|
parts := strings.Split(s, ";")
|
|
if len(parts) != 3 {
|
|
return 0, 0, errors.New(fmt.Sprintf("Expected length of 3: VALUE: %s", parts))
|
|
}
|
|
limit, err := strconv.Atoi(parts[0])
|
|
if err != nil {
|
|
return 0, 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %s", parts[0]))
|
|
}
|
|
policyParts := strings.Split(parts[1], "=")
|
|
if len(policyParts) != 2 {
|
|
return 0, 0, errors.New(fmt.Sprintf("Expected length of 2: VALUE: %s", policyParts))
|
|
}
|
|
period, err := strconv.Atoi(policyParts[1])
|
|
if err != nil {
|
|
return 0, 0, errors.New(fmt.Sprintf("Expected a number: VALUE: %d", period))
|
|
}
|
|
return limit, period, nil
|
|
}
|
|
}
|
|
|
|
return 0, 0, errors.New(fmt.Sprintf("Could not parse header, STAT: %s, HEADER: %s", stat, rateLimitPolicyHeader))
|
|
}
|