mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
kms/openbao: refresh the AppRole token before it expires (#12734)
* kms/openbao: refresh the AppRole token before it expires The provider logged in with AppRole once, when it was created, and never renewed the token. After the token's TTL every SSE-KMS upload, and every read of an SSE-KMS object whose data key was not cached, failed with "permission denied" (HTTP 500 to the client) until the S3 server was restarted. With AppRole authentication the provider now logs in again in the background once two thirds of the token's TTL have passed, and retries a failed login every 5 s. Each login is bounded by the client's timeout, or by 30 s when request_timeout is not positive. Close stops the refresh. A static token, or an AppRole token without a TTL, is left as it is. * kms: keep the OpenBao token refresh across configuration reloads - AddKMSProvider and InitializeGlobalKMS create a provider instance of their own with the new NewProvider instead of the registry's cached one: a reload closed the cached instance and then got it back, which now stopped its token refresh. The new instance is created before the old one is closed, so a configuration that fails leaves the current provider working - openbao: time each refresh from the start of the login that issued the token, since its TTL runs from then rather than from the response - tests: a reload gives a new instance and closes the old one, a failed reload keeps the old one; the refresh survives a reload; a slow login response * kms: create a replacement provider outside the manager lock AddKMSProvider ran the provider factory, which for OpenBao is a login that may wait for its timeout, while holding the manager lock, so every lookup, also of unrelated providers, waited for it. The new provider is now created before the lock is taken; the lock covers only the swap, and the replaced provider is closed after it. Test: a lookup while another provider is being created.
This commit is contained in:
1 parent
94de65b3b5
commit
3db54ee7e5
5 files changed
+471
-21
No files matched your search
+16
-13
@@ -124,7 +124,7 @@ func InitializeGlobalKMS(config *KMSConfig) error {
|
||||
providerConfig = &configAdapter{config: config.Config}
|
||||
}
|
||||
|
||||
provider, err := GetProvider(config.Provider, providerConfig)
|
||||
provider, err := NewProvider(config.Provider, providerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -208,19 +208,12 @@ func (km *KMSManager) AddKMSProvider(name string, config *KMSConfig) error {
|
||||
return fmt.Errorf("KMS configuration cannot be nil")
|
||||
}
|
||||
|
||||
km.mu.Lock()
|
||||
defer km.mu.Unlock()
|
||||
|
||||
// Close existing provider if it exists
|
||||
if existingProvider, exists := km.providers[name]; exists {
|
||||
if err := existingProvider.Close(); err != nil {
|
||||
glog.Errorf("Failed to close existing KMS provider %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create new provider instance
|
||||
// Create a new provider instance of its own: a cached registry instance would be the
|
||||
// one closed below and then handed back still in use. It is created before the lock is
|
||||
// taken and the current provider closed, so a slow configuration does not hold up other
|
||||
// lookups and one that fails leaves the current provider working.
|
||||
configAdapter := &configAdapter{config: config.Config}
|
||||
provider, err := GetProvider(config.Provider, configAdapter)
|
||||
provider, err := NewProvider(config.Provider, configAdapter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create KMS provider %s: %w", name, err)
|
||||
}
|
||||
@@ -230,8 +223,18 @@ func (km *KMSManager) AddKMSProvider(name string, config *KMSConfig) error {
|
||||
provider = wrapWithCacheIfEnabled(provider, config)
|
||||
|
||||
// Store provider and configuration
|
||||
km.mu.Lock()
|
||||
existingProvider, exists := km.providers[name]
|
||||
km.providers[name] = provider
|
||||
km.configs[name] = config
|
||||
km.mu.Unlock()
|
||||
|
||||
// Close the provider it replaces
|
||||
if exists {
|
||||
if err := existingProvider.Close(); err != nil {
|
||||
glog.Errorf("Failed to close existing KMS provider %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Added KMS provider %s (type: %s)", name, config.Provider)
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package kms
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// closeTracker is a KMSProvider that records whether it was closed.
|
||||
type closeTracker struct {
|
||||
KMSProvider
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (c *closeTracker) Close() error {
|
||||
c.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestAddKMSProviderReplacesTheInstance checks a configuration reload: the provider gets a
|
||||
// new instance of its own and the old one is closed, while a reload that fails keeps the
|
||||
// current instance open and in use.
|
||||
func TestAddKMSProviderReplacesTheInstance(t *testing.T) {
|
||||
var created []*closeTracker
|
||||
fail := false
|
||||
RegisterProvider("replace-test", func(util.Configuration) (KMSProvider, error) {
|
||||
if fail {
|
||||
return nil, errors.New("bad configuration")
|
||||
}
|
||||
p := &closeTracker{KMSProvider: newCountingKMSProvider()}
|
||||
created = append(created, p)
|
||||
return p, nil
|
||||
})
|
||||
mgr := &KMSManager{
|
||||
providers: make(map[string]KMSProvider),
|
||||
configs: make(map[string]*KMSConfig),
|
||||
bucketKMS: make(map[string]string),
|
||||
}
|
||||
add := func() error { return mgr.AddKMSProvider("p", &KMSConfig{Provider: "replace-test"}) }
|
||||
current := func() KMSProvider {
|
||||
p, err := mgr.GetKMSProviderByName("p")
|
||||
if err != nil {
|
||||
t.Fatalf("GetKMSProviderByName: %v", err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
if err := add(); err != nil {
|
||||
t.Fatalf("AddKMSProvider: %v", err)
|
||||
}
|
||||
if err := add(); err != nil {
|
||||
t.Fatalf("AddKMSProvider again: %v", err)
|
||||
}
|
||||
if len(created) != 2 || !created[0].closed || created[1].closed || current() != created[1] {
|
||||
t.Fatalf("after a reload: %d instances, first closed %v, second closed %v; want a new open instance in use",
|
||||
len(created), created[0].closed, created[len(created)-1].closed)
|
||||
}
|
||||
|
||||
fail = true
|
||||
if err := add(); err == nil {
|
||||
t.Fatal("AddKMSProvider with a failing configuration succeeded")
|
||||
}
|
||||
if created[1].closed || current() != created[1] {
|
||||
t.Fatal("a failed reload closed or replaced the provider in use")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAddKMSProviderDoesNotBlockLookups checks that creating a provider, which may wait on
|
||||
// a remote login, does not hold up lookups of the providers already configured.
|
||||
func TestAddKMSProviderDoesNotBlockLookups(t *testing.T) {
|
||||
RegisterProvider("lookup-test", func(util.Configuration) (KMSProvider, error) {
|
||||
return newCountingKMSProvider(), nil
|
||||
})
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
RegisterProvider("slow-test", func(util.Configuration) (KMSProvider, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return newCountingKMSProvider(), nil
|
||||
})
|
||||
mgr := &KMSManager{
|
||||
providers: make(map[string]KMSProvider),
|
||||
configs: make(map[string]*KMSConfig),
|
||||
bucketKMS: make(map[string]string),
|
||||
}
|
||||
if err := mgr.AddKMSProvider("ready", &KMSConfig{Provider: "lookup-test"}); err != nil {
|
||||
t.Fatalf("AddKMSProvider: %v", err)
|
||||
}
|
||||
|
||||
added := make(chan error, 1)
|
||||
go func() { added <- mgr.AddKMSProvider("slow", &KMSConfig{Provider: "slow-test"}) }()
|
||||
<-started
|
||||
looked := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := mgr.GetKMSProviderByName("ready")
|
||||
looked <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-looked:
|
||||
if err != nil {
|
||||
t.Fatalf("GetKMSProviderByName: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("a lookup waited for a provider being created")
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err := <-added; err != nil {
|
||||
t.Fatalf("AddKMSProvider (slow): %v", err)
|
||||
}
|
||||
}
|
||||
@@ -27,8 +27,18 @@ type OpenBaoKMSProvider struct {
|
||||
client *vault.Client
|
||||
transitPath string // Transit engine mount path (default: "transit")
|
||||
address string
|
||||
|
||||
stopRefresh context.CancelFunc // stops the AppRole token refresh; nil without one
|
||||
refreshDone chan struct{}
|
||||
}
|
||||
|
||||
// refreshRetryInterval is how soon a failed AppRole login is tried again.
|
||||
// Variables so that tests can shorten them.
|
||||
var (
|
||||
refreshRetryInterval = 5 * time.Second
|
||||
refreshLoginTimeout = 30 * time.Second // when request_timeout leaves the client without one
|
||||
)
|
||||
|
||||
// OpenBaoKMSConfig contains configuration for the OpenBao/Vault KMS provider
|
||||
type OpenBaoKMSConfig struct {
|
||||
Address string `json:"address"` // Vault address (e.g., "http://localhost:8200")
|
||||
@@ -103,11 +113,13 @@ func NewOpenBaoKMSProvider(config util.Configuration) (seaweedkms.KMSProvider, e
|
||||
}
|
||||
|
||||
// Authenticate
|
||||
var ttl time.Duration
|
||||
loginStart := time.Now() // the token's TTL runs from when OpenBao issues it, not from its response
|
||||
if token != "" {
|
||||
client.SetToken(token)
|
||||
glog.V(1).Infof("OpenBao KMS: Using token authentication")
|
||||
} else if roleID != "" && secretID != "" {
|
||||
if err := authenticateAppRole(client, roleID, secretID); err != nil {
|
||||
if ttl, err = authenticateAppRole(context.Background(), client, roleID, secretID); err != nil {
|
||||
return nil, fmt.Errorf("failed to authenticate with AppRole: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("OpenBao KMS: Using AppRole authentication")
|
||||
@@ -120,29 +132,72 @@ func NewOpenBaoKMSProvider(config util.Configuration) (seaweedkms.KMSProvider, e
|
||||
transitPath: transitPath,
|
||||
address: address,
|
||||
}
|
||||
if ttl > 0 {
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
provider.stopRefresh = stop
|
||||
provider.refreshDone = make(chan struct{})
|
||||
go provider.refreshAppRoleToken(ctx, roleID, secretID, loginStart, ttl)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("OpenBao/Vault KMS provider initialized at %s", address)
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
// authenticateAppRole authenticates using AppRole method
|
||||
func authenticateAppRole(client *vault.Client, roleID, secretID string) error {
|
||||
// authenticateAppRole authenticates using AppRole method and returns the token's TTL
|
||||
func authenticateAppRole(ctx context.Context, client *vault.Client, roleID, secretID string) (time.Duration, error) {
|
||||
data := map[string]interface{}{
|
||||
"role_id": roleID,
|
||||
"secret_id": secretID,
|
||||
}
|
||||
|
||||
secret, err := client.Logical().Write("auth/approle/login", data)
|
||||
secret, err := client.Logical().WriteWithContext(ctx, "auth/approle/login", data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AppRole authentication failed: %w", err)
|
||||
return 0, fmt.Errorf("AppRole authentication failed: %w", err)
|
||||
}
|
||||
|
||||
if secret == nil || secret.Auth == nil {
|
||||
return fmt.Errorf("AppRole authentication returned empty token")
|
||||
return 0, fmt.Errorf("AppRole authentication returned empty token")
|
||||
}
|
||||
|
||||
client.SetToken(secret.Auth.ClientToken)
|
||||
return nil
|
||||
return time.Duration(secret.Auth.LeaseDuration) * time.Second, nil
|
||||
}
|
||||
|
||||
// refreshAppRoleToken logs in again once two thirds of the token's TTL have passed since the
|
||||
// login that issued it started, and retries a failed login every refreshRetryInterval, until
|
||||
// ctx is cancelled. Nothing renews an AppRole token, so without this every request fails
|
||||
// once it expires.
|
||||
func (p *OpenBaoKMSProvider) refreshAppRoleToken(ctx context.Context, roleID, secretID string, issued time.Time, ttl time.Duration) {
|
||||
defer close(p.refreshDone)
|
||||
timeout := p.client.ClientTimeout()
|
||||
if timeout <= 0 {
|
||||
timeout = refreshLoginTimeout
|
||||
}
|
||||
timer := time.NewTimer(time.Until(issued.Add(ttl * 2 / 3)))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
loginCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
issued = time.Now()
|
||||
newTTL, err := authenticateAppRole(loginCtx, p.client, roleID, secretID)
|
||||
cancel()
|
||||
switch {
|
||||
case err != nil:
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
glog.Warningf("OpenBao KMS: refreshing the AppRole token failed, retrying in %v: %v", refreshRetryInterval, err)
|
||||
timer.Reset(refreshRetryInterval)
|
||||
case newTTL <= 0:
|
||||
return // the new token does not expire
|
||||
default:
|
||||
timer.Reset(time.Until(issued.Add(newTTL * 2 / 3)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateDataKey generates a new data encryption key using OpenBao/Vault Transit
|
||||
@@ -361,7 +416,10 @@ func (p *OpenBaoKMSProvider) GetKeyID(ctx context.Context, keyIdentifier string)
|
||||
|
||||
// Close cleans up any resources used by the provider
|
||||
func (p *OpenBaoKMSProvider) Close() error {
|
||||
// OpenBao/Vault client doesn't require explicit cleanup
|
||||
if p.stopRefresh != nil {
|
||||
p.stopRefresh()
|
||||
<-p.refreshDone
|
||||
}
|
||||
glog.V(2).Infof("OpenBao/Vault KMS provider closed")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
package openbao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seaweedkms "github.com/seaweedfs/seaweedfs/weed/kms"
|
||||
)
|
||||
|
||||
// mapConfig is a util.Configuration backed by a map.
|
||||
type mapConfig map[string]interface{}
|
||||
|
||||
func (c mapConfig) GetString(key string) string { s, _ := c[key].(string); return s }
|
||||
func (c mapConfig) GetBool(key string) bool { b, _ := c[key].(bool); return b }
|
||||
func (c mapConfig) GetInt(key string) int { i, _ := c[key].(int); return i }
|
||||
func (c mapConfig) GetStringSlice(key string) []string { s, _ := c[key].([]string); return s }
|
||||
func (c mapConfig) SetDefault(string, interface{}) {}
|
||||
|
||||
// fakeBao issues a token that expires after ttl seconds on every AppRole login and accepts
|
||||
// transit requests only with a token that has not expired, like OpenBao.
|
||||
type fakeBao struct {
|
||||
mu sync.Mutex
|
||||
ttl int // 0 issues tokens that never expire
|
||||
logins int
|
||||
failedLogins int
|
||||
failLogins int // how many of the next logins to refuse
|
||||
hangLogins bool // logins after the first hang until their client leaves
|
||||
loginDelay time.Duration // logins after the first answer this late; the token's TTL runs from the request
|
||||
loginHung chan struct{} // signalled when such a login is given up
|
||||
expiry map[string]time.Time
|
||||
}
|
||||
|
||||
func (f *fakeBao) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.Copy(io.Discard, r.Body) // only a fully read request notices its client leaving
|
||||
f.mu.Lock()
|
||||
if r.URL.Path == "/v1/auth/approle/login" {
|
||||
if f.hangLogins && f.logins > 0 {
|
||||
hung := f.loginHung
|
||||
f.mu.Unlock()
|
||||
<-r.Context().Done()
|
||||
hung <- struct{}{}
|
||||
return
|
||||
}
|
||||
if f.failLogins > 0 {
|
||||
f.failLogins--
|
||||
f.failedLogins++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(`{"errors":["invalid secret id"]}`))
|
||||
return
|
||||
}
|
||||
f.logins++
|
||||
token := fmt.Sprintf("token-%d", f.logins)
|
||||
if f.expiry == nil {
|
||||
f.expiry = map[string]time.Time{}
|
||||
}
|
||||
f.expiry[token] = time.Time{}
|
||||
if f.ttl > 0 {
|
||||
f.expiry[token] = time.Now().Add(time.Duration(f.ttl) * time.Second)
|
||||
}
|
||||
ttl := f.ttl
|
||||
delay := time.Duration(0)
|
||||
if f.logins > 1 {
|
||||
delay = f.loginDelay
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(delay)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"auth": map[string]interface{}{"client_token": token, "lease_duration": ttl},
|
||||
})
|
||||
return
|
||||
}
|
||||
expires, issued := f.expiry[r.Header.Get("X-Vault-Token")]
|
||||
f.mu.Unlock()
|
||||
if !issued || (!expires.IsZero() && time.Now().After(expires)) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
w.Write([]byte(`{"errors":["permission denied"]}`))
|
||||
return
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": map[string]interface{}{"ciphertext": "vault:v1:c2VhbGVk"},
|
||||
})
|
||||
}
|
||||
|
||||
func (f *fakeBao) counts() (logins, failedLogins int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.logins, f.failedLogins
|
||||
}
|
||||
|
||||
func newTestProvider(t *testing.T, bao *fakeBao, config mapConfig) *OpenBaoKMSProvider {
|
||||
t.Helper()
|
||||
t.Setenv("VAULT_AGENT_ADDR", "") // would take the place of the test server's address
|
||||
server := httptest.NewServer(bao)
|
||||
t.Cleanup(server.Close)
|
||||
config["address"] = server.URL
|
||||
provider, err := NewOpenBaoKMSProvider(config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenBaoKMSProvider: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { provider.Close() })
|
||||
return provider.(*OpenBaoKMSProvider)
|
||||
}
|
||||
|
||||
func appRoleConfig() mapConfig {
|
||||
return mapConfig{"role_id": "role", "secret_id": "secret"}
|
||||
}
|
||||
|
||||
// setForTest changes a package variable for the length of the test.
|
||||
func setForTest[T any](t *testing.T, v *T, value T) {
|
||||
saved := *v
|
||||
*v = value
|
||||
t.Cleanup(func() { *v = saved })
|
||||
}
|
||||
|
||||
// keepRequesting sends a request every 100ms for d and fails the test on the first error.
|
||||
func keepRequesting(t *testing.T, provider seaweedkms.KMSProvider, d time.Duration) {
|
||||
t.Helper()
|
||||
start := time.Now()
|
||||
for time.Since(start) < d {
|
||||
_, err := provider.GenerateDataKey(context.Background(), &seaweedkms.GenerateDataKeyRequest{
|
||||
KeyID: "test-key",
|
||||
KeySpec: seaweedkms.KeySpecAES256,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("request %v after the start: %v", time.Since(start).Round(time.Millisecond), err)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// closeWithin closes the provider and fails the test if that takes more than a few seconds.
|
||||
func closeWithin(t *testing.T, provider *OpenBaoKMSProvider) {
|
||||
t.Helper()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
provider.Close()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close did not return")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppRoleTokenIsRefreshedBeforeItExpires(t *testing.T) {
|
||||
bao := &fakeBao{ttl: 2}
|
||||
provider := newTestProvider(t, bao, appRoleConfig())
|
||||
|
||||
keepRequesting(t, provider, 3500*time.Millisecond)
|
||||
if logins, _ := bao.counts(); logins < 3 {
|
||||
t.Fatalf("logins = %d, want at least 3 in 3.5 s of 2 s tokens", logins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshIsTimedFromTheLoginStart(t *testing.T) {
|
||||
// 3 s tokens whose login answers after 0.75 s: timed from the response, the second
|
||||
// refresh would start at 4.75 s and its token arrive at 5.5 s, after the token issued at
|
||||
// 2 s expired at 5 s; timed from the login start it arrives at 4.75 s.
|
||||
bao := &fakeBao{ttl: 3, loginDelay: 750 * time.Millisecond}
|
||||
provider := newTestProvider(t, bao, appRoleConfig())
|
||||
|
||||
keepRequesting(t, provider, 6*time.Second)
|
||||
}
|
||||
|
||||
func TestRefreshSurvivesAConfigurationReload(t *testing.T) {
|
||||
t.Setenv("VAULT_AGENT_ADDR", "")
|
||||
bao := &fakeBao{ttl: 2}
|
||||
server := httptest.NewServer(bao)
|
||||
t.Cleanup(server.Close)
|
||||
manager := seaweedkms.InitializeKMSManager()
|
||||
t.Cleanup(func() { manager.RemoveKMSProvider("reload-test") })
|
||||
config := &seaweedkms.KMSConfig{Provider: "openbao", Config: map[string]interface{}{
|
||||
"address": server.URL, "role_id": "role", "secret_id": "secret",
|
||||
}}
|
||||
|
||||
// A reload closes the provider it replaces; the one that stays must keep refreshing.
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := manager.AddKMSProvider("reload-test", config); err != nil {
|
||||
t.Fatalf("AddKMSProvider: %v", err)
|
||||
}
|
||||
}
|
||||
provider, err := manager.GetKMSProviderByName("reload-test")
|
||||
if err != nil {
|
||||
t.Fatalf("GetKMSProviderByName: %v", err)
|
||||
}
|
||||
keepRequesting(t, provider, 3500*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestFailedRefreshIsRetried(t *testing.T) {
|
||||
setForTest(t, &refreshRetryInterval, 100*time.Millisecond)
|
||||
bao := &fakeBao{ttl: 2}
|
||||
provider := newTestProvider(t, bao, appRoleConfig())
|
||||
bao.mu.Lock()
|
||||
bao.failLogins = 2
|
||||
bao.mu.Unlock()
|
||||
|
||||
// The refresh at 1.33 s fails twice and succeeds 200 ms later, before the token expires.
|
||||
keepRequesting(t, provider, 3*time.Second)
|
||||
if logins, failed := bao.counts(); failed != 2 || logins < 2 {
|
||||
t.Fatalf("logins = %d, failed logins = %d, want at least 2 and 2", logins, failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseStopsTheRefresh(t *testing.T) {
|
||||
bao := &fakeBao{ttl: 1}
|
||||
provider := newTestProvider(t, bao, appRoleConfig())
|
||||
|
||||
closeWithin(t, provider)
|
||||
time.Sleep(time.Second) // past the refresh due at 0.67 s
|
||||
if logins, _ := bao.counts(); logins != 1 {
|
||||
t.Fatalf("logins = %d, want 1", logins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHangingRefreshIsGivenUpAndRetried(t *testing.T) {
|
||||
setForTest(t, &refreshRetryInterval, 100*time.Millisecond)
|
||||
setForTest(t, &refreshLoginTimeout, 100*time.Millisecond)
|
||||
config := appRoleConfig()
|
||||
config["request_timeout"] = -1 // leaves the client without a timeout
|
||||
bao := &fakeBao{ttl: 1, hangLogins: true, loginHung: make(chan struct{}, 10)}
|
||||
provider := newTestProvider(t, bao, config)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
select {
|
||||
case <-bao.loginHung:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("%d hanging refresh logins were given up, want 2", i)
|
||||
}
|
||||
}
|
||||
closeWithin(t, provider)
|
||||
}
|
||||
|
||||
func TestNoRefreshWithoutAnExpiringToken(t *testing.T) {
|
||||
for name, config := range map[string]mapConfig{
|
||||
"static token": {"token": "static"},
|
||||
"AppRole without a TTL": appRoleConfig(),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
provider := newTestProvider(t, &fakeBao{ttl: 0}, config)
|
||||
if provider.stopRefresh != nil {
|
||||
t.Fatal("a refresh was started")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,28 @@ func (r *ProviderRegistry) GetProvider(name string, config util.Configuration) (
|
||||
return instance, nil
|
||||
}
|
||||
|
||||
// NewProvider creates a new instance of a registered KMS provider. Unlike GetProvider it
|
||||
// neither caches nor shares the instance: the caller owns it and closes it.
|
||||
func NewProvider(name string, config util.Configuration) (KMSProvider, error) {
|
||||
return defaultRegistry.NewProvider(name, config)
|
||||
}
|
||||
|
||||
// NewProvider creates a new, unshared instance of a registered KMS provider
|
||||
func (r *ProviderRegistry) NewProvider(name string, config util.Configuration) (KMSProvider, error) {
|
||||
r.mu.RLock()
|
||||
factory, exists := r.providers[name]
|
||||
r.mu.RUnlock()
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("KMS provider '%s' not registered", name)
|
||||
}
|
||||
|
||||
instance, err := factory(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create KMS provider '%s': %v", name, err)
|
||||
}
|
||||
return instance, nil
|
||||
}
|
||||
|
||||
// ListProviders returns the names of all registered providers
|
||||
func ListProviders() []string {
|
||||
return defaultRegistry.ListProviders()
|
||||
|
||||
Reference in new issue
Block a user