diff --git a/weed/kms/config.go b/weed/kms/config.go index f1c66c58fb..fcb65eef6f 100644 --- a/weed/kms/config.go +++ b/weed/kms/config.go @@ -124,7 +124,7 @@ func InitializeGlobalKMS(config *KMSConfig) error { providerConfig = &configAdapter{config: config.Config} } - provider, err := GetProvider(config.Provider, providerConfig) + provider, err := NewProvider(config.Provider, providerConfig) if err != nil { return err } @@ -208,19 +208,12 @@ func (km *KMSManager) AddKMSProvider(name string, config *KMSConfig) error { return fmt.Errorf("KMS configuration cannot be nil") } - km.mu.Lock() - defer km.mu.Unlock() - - // Close existing provider if it exists - if existingProvider, exists := km.providers[name]; exists { - if err := existingProvider.Close(); err != nil { - glog.Errorf("Failed to close existing KMS provider %s: %v", name, err) - } - } - - // Create new provider instance + // Create a new provider instance of its own: a cached registry instance would be the + // one closed below and then handed back still in use. It is created before the lock is + // taken and the current provider closed, so a slow configuration does not hold up other + // lookups and one that fails leaves the current provider working. configAdapter := &configAdapter{config: config.Config} - provider, err := GetProvider(config.Provider, configAdapter) + provider, err := NewProvider(config.Provider, configAdapter) if err != nil { return fmt.Errorf("failed to create KMS provider %s: %w", name, err) } @@ -230,8 +223,18 @@ func (km *KMSManager) AddKMSProvider(name string, config *KMSConfig) error { provider = wrapWithCacheIfEnabled(provider, config) // Store provider and configuration + km.mu.Lock() + existingProvider, exists := km.providers[name] km.providers[name] = provider km.configs[name] = config + km.mu.Unlock() + + // Close the provider it replaces + if exists { + if err := existingProvider.Close(); err != nil { + glog.Errorf("Failed to close existing KMS provider %s: %v", name, err) + } + } glog.V(1).Infof("Added KMS provider %s (type: %s)", name, config.Provider) return nil diff --git a/weed/kms/config_test.go b/weed/kms/config_test.go new file mode 100644 index 0000000000..6b8a474ea1 --- /dev/null +++ b/weed/kms/config_test.go @@ -0,0 +1,112 @@ +package kms + +import ( + "errors" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// closeTracker is a KMSProvider that records whether it was closed. +type closeTracker struct { + KMSProvider + closed bool +} + +func (c *closeTracker) Close() error { + c.closed = true + return nil +} + +// TestAddKMSProviderReplacesTheInstance checks a configuration reload: the provider gets a +// new instance of its own and the old one is closed, while a reload that fails keeps the +// current instance open and in use. +func TestAddKMSProviderReplacesTheInstance(t *testing.T) { + var created []*closeTracker + fail := false + RegisterProvider("replace-test", func(util.Configuration) (KMSProvider, error) { + if fail { + return nil, errors.New("bad configuration") + } + p := &closeTracker{KMSProvider: newCountingKMSProvider()} + created = append(created, p) + return p, nil + }) + mgr := &KMSManager{ + providers: make(map[string]KMSProvider), + configs: make(map[string]*KMSConfig), + bucketKMS: make(map[string]string), + } + add := func() error { return mgr.AddKMSProvider("p", &KMSConfig{Provider: "replace-test"}) } + current := func() KMSProvider { + p, err := mgr.GetKMSProviderByName("p") + if err != nil { + t.Fatalf("GetKMSProviderByName: %v", err) + } + return p + } + + if err := add(); err != nil { + t.Fatalf("AddKMSProvider: %v", err) + } + if err := add(); err != nil { + t.Fatalf("AddKMSProvider again: %v", err) + } + if len(created) != 2 || !created[0].closed || created[1].closed || current() != created[1] { + t.Fatalf("after a reload: %d instances, first closed %v, second closed %v; want a new open instance in use", + len(created), created[0].closed, created[len(created)-1].closed) + } + + fail = true + if err := add(); err == nil { + t.Fatal("AddKMSProvider with a failing configuration succeeded") + } + if created[1].closed || current() != created[1] { + t.Fatal("a failed reload closed or replaced the provider in use") + } +} + +// TestAddKMSProviderDoesNotBlockLookups checks that creating a provider, which may wait on +// a remote login, does not hold up lookups of the providers already configured. +func TestAddKMSProviderDoesNotBlockLookups(t *testing.T) { + RegisterProvider("lookup-test", func(util.Configuration) (KMSProvider, error) { + return newCountingKMSProvider(), nil + }) + started, release := make(chan struct{}), make(chan struct{}) + RegisterProvider("slow-test", func(util.Configuration) (KMSProvider, error) { + close(started) + <-release + return newCountingKMSProvider(), nil + }) + mgr := &KMSManager{ + providers: make(map[string]KMSProvider), + configs: make(map[string]*KMSConfig), + bucketKMS: make(map[string]string), + } + if err := mgr.AddKMSProvider("ready", &KMSConfig{Provider: "lookup-test"}); err != nil { + t.Fatalf("AddKMSProvider: %v", err) + } + + added := make(chan error, 1) + go func() { added <- mgr.AddKMSProvider("slow", &KMSConfig{Provider: "slow-test"}) }() + <-started + looked := make(chan error, 1) + go func() { + _, err := mgr.GetKMSProviderByName("ready") + looked <- err + }() + select { + case err := <-looked: + if err != nil { + t.Fatalf("GetKMSProviderByName: %v", err) + } + case <-time.After(time.Second): + t.Fatal("a lookup waited for a provider being created") + } + + close(release) + if err := <-added; err != nil { + t.Fatalf("AddKMSProvider (slow): %v", err) + } +} diff --git a/weed/kms/openbao/openbao_kms.go b/weed/kms/openbao/openbao_kms.go index 259a689b3a..4b3335223e 100644 --- a/weed/kms/openbao/openbao_kms.go +++ b/weed/kms/openbao/openbao_kms.go @@ -27,8 +27,18 @@ type OpenBaoKMSProvider struct { client *vault.Client transitPath string // Transit engine mount path (default: "transit") address string + + stopRefresh context.CancelFunc // stops the AppRole token refresh; nil without one + refreshDone chan struct{} } +// refreshRetryInterval is how soon a failed AppRole login is tried again. +// Variables so that tests can shorten them. +var ( + refreshRetryInterval = 5 * time.Second + refreshLoginTimeout = 30 * time.Second // when request_timeout leaves the client without one +) + // OpenBaoKMSConfig contains configuration for the OpenBao/Vault KMS provider type OpenBaoKMSConfig struct { Address string `json:"address"` // Vault address (e.g., "http://localhost:8200") @@ -103,11 +113,13 @@ func NewOpenBaoKMSProvider(config util.Configuration) (seaweedkms.KMSProvider, e } // Authenticate + var ttl time.Duration + loginStart := time.Now() // the token's TTL runs from when OpenBao issues it, not from its response if token != "" { client.SetToken(token) glog.V(1).Infof("OpenBao KMS: Using token authentication") } else if roleID != "" && secretID != "" { - if err := authenticateAppRole(client, roleID, secretID); err != nil { + if ttl, err = authenticateAppRole(context.Background(), client, roleID, secretID); err != nil { return nil, fmt.Errorf("failed to authenticate with AppRole: %w", err) } glog.V(1).Infof("OpenBao KMS: Using AppRole authentication") @@ -120,29 +132,72 @@ func NewOpenBaoKMSProvider(config util.Configuration) (seaweedkms.KMSProvider, e transitPath: transitPath, address: address, } + if ttl > 0 { + ctx, stop := context.WithCancel(context.Background()) + provider.stopRefresh = stop + provider.refreshDone = make(chan struct{}) + go provider.refreshAppRoleToken(ctx, roleID, secretID, loginStart, ttl) + } glog.V(1).Infof("OpenBao/Vault KMS provider initialized at %s", address) return provider, nil } -// authenticateAppRole authenticates using AppRole method -func authenticateAppRole(client *vault.Client, roleID, secretID string) error { +// authenticateAppRole authenticates using AppRole method and returns the token's TTL +func authenticateAppRole(ctx context.Context, client *vault.Client, roleID, secretID string) (time.Duration, error) { data := map[string]interface{}{ "role_id": roleID, "secret_id": secretID, } - secret, err := client.Logical().Write("auth/approle/login", data) + secret, err := client.Logical().WriteWithContext(ctx, "auth/approle/login", data) if err != nil { - return fmt.Errorf("AppRole authentication failed: %w", err) + return 0, fmt.Errorf("AppRole authentication failed: %w", err) } if secret == nil || secret.Auth == nil { - return fmt.Errorf("AppRole authentication returned empty token") + return 0, fmt.Errorf("AppRole authentication returned empty token") } client.SetToken(secret.Auth.ClientToken) - return nil + return time.Duration(secret.Auth.LeaseDuration) * time.Second, nil +} + +// refreshAppRoleToken logs in again once two thirds of the token's TTL have passed since the +// login that issued it started, and retries a failed login every refreshRetryInterval, until +// ctx is cancelled. Nothing renews an AppRole token, so without this every request fails +// once it expires. +func (p *OpenBaoKMSProvider) refreshAppRoleToken(ctx context.Context, roleID, secretID string, issued time.Time, ttl time.Duration) { + defer close(p.refreshDone) + timeout := p.client.ClientTimeout() + if timeout <= 0 { + timeout = refreshLoginTimeout + } + timer := time.NewTimer(time.Until(issued.Add(ttl * 2 / 3))) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + loginCtx, cancel := context.WithTimeout(ctx, timeout) + issued = time.Now() + newTTL, err := authenticateAppRole(loginCtx, p.client, roleID, secretID) + cancel() + switch { + case err != nil: + if ctx.Err() != nil { + return + } + glog.Warningf("OpenBao KMS: refreshing the AppRole token failed, retrying in %v: %v", refreshRetryInterval, err) + timer.Reset(refreshRetryInterval) + case newTTL <= 0: + return // the new token does not expire + default: + timer.Reset(time.Until(issued.Add(newTTL * 2 / 3))) + } + } } // GenerateDataKey generates a new data encryption key using OpenBao/Vault Transit @@ -361,7 +416,10 @@ func (p *OpenBaoKMSProvider) GetKeyID(ctx context.Context, keyIdentifier string) // Close cleans up any resources used by the provider func (p *OpenBaoKMSProvider) Close() error { - // OpenBao/Vault client doesn't require explicit cleanup + if p.stopRefresh != nil { + p.stopRefresh() + <-p.refreshDone + } glog.V(2).Infof("OpenBao/Vault KMS provider closed") return nil } diff --git a/weed/kms/openbao/openbao_kms_test.go b/weed/kms/openbao/openbao_kms_test.go new file mode 100644 index 0000000000..abc352c3b6 --- /dev/null +++ b/weed/kms/openbao/openbao_kms_test.go @@ -0,0 +1,255 @@ +package openbao + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + seaweedkms "github.com/seaweedfs/seaweedfs/weed/kms" +) + +// mapConfig is a util.Configuration backed by a map. +type mapConfig map[string]interface{} + +func (c mapConfig) GetString(key string) string { s, _ := c[key].(string); return s } +func (c mapConfig) GetBool(key string) bool { b, _ := c[key].(bool); return b } +func (c mapConfig) GetInt(key string) int { i, _ := c[key].(int); return i } +func (c mapConfig) GetStringSlice(key string) []string { s, _ := c[key].([]string); return s } +func (c mapConfig) SetDefault(string, interface{}) {} + +// fakeBao issues a token that expires after ttl seconds on every AppRole login and accepts +// transit requests only with a token that has not expired, like OpenBao. +type fakeBao struct { + mu sync.Mutex + ttl int // 0 issues tokens that never expire + logins int + failedLogins int + failLogins int // how many of the next logins to refuse + hangLogins bool // logins after the first hang until their client leaves + loginDelay time.Duration // logins after the first answer this late; the token's TTL runs from the request + loginHung chan struct{} // signalled when such a login is given up + expiry map[string]time.Time +} + +func (f *fakeBao) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.Copy(io.Discard, r.Body) // only a fully read request notices its client leaving + f.mu.Lock() + if r.URL.Path == "/v1/auth/approle/login" { + if f.hangLogins && f.logins > 0 { + hung := f.loginHung + f.mu.Unlock() + <-r.Context().Done() + hung <- struct{}{} + return + } + if f.failLogins > 0 { + f.failLogins-- + f.failedLogins++ + f.mu.Unlock() + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"errors":["invalid secret id"]}`)) + return + } + f.logins++ + token := fmt.Sprintf("token-%d", f.logins) + if f.expiry == nil { + f.expiry = map[string]time.Time{} + } + f.expiry[token] = time.Time{} + if f.ttl > 0 { + f.expiry[token] = time.Now().Add(time.Duration(f.ttl) * time.Second) + } + ttl := f.ttl + delay := time.Duration(0) + if f.logins > 1 { + delay = f.loginDelay + } + f.mu.Unlock() + time.Sleep(delay) + json.NewEncoder(w).Encode(map[string]interface{}{ + "auth": map[string]interface{}{"client_token": token, "lease_duration": ttl}, + }) + return + } + expires, issued := f.expiry[r.Header.Get("X-Vault-Token")] + f.mu.Unlock() + if !issued || (!expires.IsZero() && time.Now().After(expires)) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(`{"errors":["permission denied"]}`)) + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "data": map[string]interface{}{"ciphertext": "vault:v1:c2VhbGVk"}, + }) +} + +func (f *fakeBao) counts() (logins, failedLogins int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.logins, f.failedLogins +} + +func newTestProvider(t *testing.T, bao *fakeBao, config mapConfig) *OpenBaoKMSProvider { + t.Helper() + t.Setenv("VAULT_AGENT_ADDR", "") // would take the place of the test server's address + server := httptest.NewServer(bao) + t.Cleanup(server.Close) + config["address"] = server.URL + provider, err := NewOpenBaoKMSProvider(config) + if err != nil { + t.Fatalf("NewOpenBaoKMSProvider: %v", err) + } + t.Cleanup(func() { provider.Close() }) + return provider.(*OpenBaoKMSProvider) +} + +func appRoleConfig() mapConfig { + return mapConfig{"role_id": "role", "secret_id": "secret"} +} + +// setForTest changes a package variable for the length of the test. +func setForTest[T any](t *testing.T, v *T, value T) { + saved := *v + *v = value + t.Cleanup(func() { *v = saved }) +} + +// keepRequesting sends a request every 100ms for d and fails the test on the first error. +func keepRequesting(t *testing.T, provider seaweedkms.KMSProvider, d time.Duration) { + t.Helper() + start := time.Now() + for time.Since(start) < d { + _, err := provider.GenerateDataKey(context.Background(), &seaweedkms.GenerateDataKeyRequest{ + KeyID: "test-key", + KeySpec: seaweedkms.KeySpecAES256, + }) + if err != nil { + t.Fatalf("request %v after the start: %v", time.Since(start).Round(time.Millisecond), err) + } + time.Sleep(100 * time.Millisecond) + } +} + +// closeWithin closes the provider and fails the test if that takes more than a few seconds. +func closeWithin(t *testing.T, provider *OpenBaoKMSProvider) { + t.Helper() + done := make(chan struct{}) + go func() { + provider.Close() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close did not return") + } +} + +func TestAppRoleTokenIsRefreshedBeforeItExpires(t *testing.T) { + bao := &fakeBao{ttl: 2} + provider := newTestProvider(t, bao, appRoleConfig()) + + keepRequesting(t, provider, 3500*time.Millisecond) + if logins, _ := bao.counts(); logins < 3 { + t.Fatalf("logins = %d, want at least 3 in 3.5 s of 2 s tokens", logins) + } +} + +func TestRefreshIsTimedFromTheLoginStart(t *testing.T) { + // 3 s tokens whose login answers after 0.75 s: timed from the response, the second + // refresh would start at 4.75 s and its token arrive at 5.5 s, after the token issued at + // 2 s expired at 5 s; timed from the login start it arrives at 4.75 s. + bao := &fakeBao{ttl: 3, loginDelay: 750 * time.Millisecond} + provider := newTestProvider(t, bao, appRoleConfig()) + + keepRequesting(t, provider, 6*time.Second) +} + +func TestRefreshSurvivesAConfigurationReload(t *testing.T) { + t.Setenv("VAULT_AGENT_ADDR", "") + bao := &fakeBao{ttl: 2} + server := httptest.NewServer(bao) + t.Cleanup(server.Close) + manager := seaweedkms.InitializeKMSManager() + t.Cleanup(func() { manager.RemoveKMSProvider("reload-test") }) + config := &seaweedkms.KMSConfig{Provider: "openbao", Config: map[string]interface{}{ + "address": server.URL, "role_id": "role", "secret_id": "secret", + }} + + // A reload closes the provider it replaces; the one that stays must keep refreshing. + for i := 0; i < 2; i++ { + if err := manager.AddKMSProvider("reload-test", config); err != nil { + t.Fatalf("AddKMSProvider: %v", err) + } + } + provider, err := manager.GetKMSProviderByName("reload-test") + if err != nil { + t.Fatalf("GetKMSProviderByName: %v", err) + } + keepRequesting(t, provider, 3500*time.Millisecond) +} + +func TestFailedRefreshIsRetried(t *testing.T) { + setForTest(t, &refreshRetryInterval, 100*time.Millisecond) + bao := &fakeBao{ttl: 2} + provider := newTestProvider(t, bao, appRoleConfig()) + bao.mu.Lock() + bao.failLogins = 2 + bao.mu.Unlock() + + // The refresh at 1.33 s fails twice and succeeds 200 ms later, before the token expires. + keepRequesting(t, provider, 3*time.Second) + if logins, failed := bao.counts(); failed != 2 || logins < 2 { + t.Fatalf("logins = %d, failed logins = %d, want at least 2 and 2", logins, failed) + } +} + +func TestCloseStopsTheRefresh(t *testing.T) { + bao := &fakeBao{ttl: 1} + provider := newTestProvider(t, bao, appRoleConfig()) + + closeWithin(t, provider) + time.Sleep(time.Second) // past the refresh due at 0.67 s + if logins, _ := bao.counts(); logins != 1 { + t.Fatalf("logins = %d, want 1", logins) + } +} + +func TestHangingRefreshIsGivenUpAndRetried(t *testing.T) { + setForTest(t, &refreshRetryInterval, 100*time.Millisecond) + setForTest(t, &refreshLoginTimeout, 100*time.Millisecond) + config := appRoleConfig() + config["request_timeout"] = -1 // leaves the client without a timeout + bao := &fakeBao{ttl: 1, hangLogins: true, loginHung: make(chan struct{}, 10)} + provider := newTestProvider(t, bao, config) + + for i := 0; i < 2; i++ { + select { + case <-bao.loginHung: + case <-time.After(5 * time.Second): + t.Fatalf("%d hanging refresh logins were given up, want 2", i) + } + } + closeWithin(t, provider) +} + +func TestNoRefreshWithoutAnExpiringToken(t *testing.T) { + for name, config := range map[string]mapConfig{ + "static token": {"token": "static"}, + "AppRole without a TTL": appRoleConfig(), + } { + t.Run(name, func(t *testing.T) { + provider := newTestProvider(t, &fakeBao{ttl: 0}, config) + if provider.stopRefresh != nil { + t.Fatal("a refresh was started") + } + }) + } +} diff --git a/weed/kms/registry.go b/weed/kms/registry.go index d1d812f710..7751e460bd 100644 --- a/weed/kms/registry.go +++ b/weed/kms/registry.go @@ -73,6 +73,28 @@ func (r *ProviderRegistry) GetProvider(name string, config util.Configuration) ( return instance, nil } +// NewProvider creates a new instance of a registered KMS provider. Unlike GetProvider it +// neither caches nor shares the instance: the caller owns it and closes it. +func NewProvider(name string, config util.Configuration) (KMSProvider, error) { + return defaultRegistry.NewProvider(name, config) +} + +// NewProvider creates a new, unshared instance of a registered KMS provider +func (r *ProviderRegistry) NewProvider(name string, config util.Configuration) (KMSProvider, error) { + r.mu.RLock() + factory, exists := r.providers[name] + r.mu.RUnlock() + if !exists { + return nil, fmt.Errorf("KMS provider '%s' not registered", name) + } + + instance, err := factory(config) + if err != nil { + return nil, fmt.Errorf("failed to create KMS provider '%s': %v", name, err) + } + return instance, nil +} + // ListProviders returns the names of all registered providers func ListProviders() []string { return defaultRegistry.ListProviders()