Files
seaweedfs/weed/kms/registry.go
T
Denis 3db54ee7e5 kms/openbao: refresh the AppRole token before it expires (#12734)
* kms/openbao: refresh the AppRole token before it expires

The provider logged in with AppRole once, when it was created, and
never renewed the token. After the token's TTL every SSE-KMS upload,
and every read of an SSE-KMS object whose data key was not cached,
failed with "permission denied" (HTTP 500 to the client) until the S3
server was restarted.

With AppRole authentication the provider now logs in again in the
background once two thirds of the token's TTL have passed, and retries
a failed login every 5 s. Each login is bounded by the client's
timeout, or by 30 s when request_timeout is not positive. Close stops
the refresh. A static token, or an AppRole token without a TTL, is left
as it is.

* kms: keep the OpenBao token refresh across configuration reloads

- AddKMSProvider and InitializeGlobalKMS create a provider instance of
  their own with the new NewProvider instead of the registry's cached
  one: a reload closed the cached instance and then got it back, which
  now stopped its token refresh. The new instance is created before the
  old one is closed, so a configuration that fails leaves the current
  provider working
- openbao: time each refresh from the start of the login that issued
  the token, since its TTL runs from then rather than from the response
- tests: a reload gives a new instance and closes the old one, a failed
  reload keeps the old one; the refresh survives a reload; a slow login
  response

* kms: create a replacement provider outside the manager lock

AddKMSProvider ran the provider factory, which for OpenBao is a login
that may wait for its timeout, while holding the manager lock, so every
lookup, also of unrelated providers, waited for it. The new provider is
now created before the lock is taken; the lock covers only the swap, and
the replaced provider is closed after it. Test: a lookup while another
provider is being created.
2026-11-18 11:07:09 -08:00

168 lines
4.6 KiB
Go

package kms
import (
"context"
"errors"
"fmt"
"sync"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// ProviderRegistry manages KMS provider implementations
type ProviderRegistry struct {
mu sync.RWMutex
providers map[string]ProviderFactory
instances map[string]KMSProvider
}
// ProviderFactory creates a new KMS provider instance
type ProviderFactory func(config util.Configuration) (KMSProvider, error)
var defaultRegistry = NewProviderRegistry()
// NewProviderRegistry creates a new provider registry
func NewProviderRegistry() *ProviderRegistry {
return &ProviderRegistry{
providers: make(map[string]ProviderFactory),
instances: make(map[string]KMSProvider),
}
}
// RegisterProvider registers a new KMS provider factory
func RegisterProvider(name string, factory ProviderFactory) {
defaultRegistry.RegisterProvider(name, factory)
}
// RegisterProvider registers a new KMS provider factory in this registry
func (r *ProviderRegistry) RegisterProvider(name string, factory ProviderFactory) {
r.mu.Lock()
defer r.mu.Unlock()
r.providers[name] = factory
}
// GetProvider returns a KMS provider instance, creating it if necessary
func GetProvider(name string, config util.Configuration) (KMSProvider, error) {
return defaultRegistry.GetProvider(name, config)
}
// GetProvider returns a KMS provider instance, creating it if necessary
func (r *ProviderRegistry) GetProvider(name string, config util.Configuration) (KMSProvider, error) {
r.mu.Lock()
defer r.mu.Unlock()
// Return existing instance if available
if instance, exists := r.instances[name]; exists {
return instance, nil
}
// Find the factory
factory, exists := r.providers[name]
if !exists {
return nil, fmt.Errorf("KMS provider '%s' not registered", name)
}
// Create new instance
instance, err := factory(config)
if err != nil {
return nil, fmt.Errorf("failed to create KMS provider '%s': %v", name, err)
}
// Cache the instance
r.instances[name] = instance
return instance, nil
}
// NewProvider creates a new instance of a registered KMS provider. Unlike GetProvider it
// neither caches nor shares the instance: the caller owns it and closes it.
func NewProvider(name string, config util.Configuration) (KMSProvider, error) {
return defaultRegistry.NewProvider(name, config)
}
// NewProvider creates a new, unshared instance of a registered KMS provider
func (r *ProviderRegistry) NewProvider(name string, config util.Configuration) (KMSProvider, error) {
r.mu.RLock()
factory, exists := r.providers[name]
r.mu.RUnlock()
if !exists {
return nil, fmt.Errorf("KMS provider '%s' not registered", name)
}
instance, err := factory(config)
if err != nil {
return nil, fmt.Errorf("failed to create KMS provider '%s': %v", name, err)
}
return instance, nil
}
// ListProviders returns the names of all registered providers
func ListProviders() []string {
return defaultRegistry.ListProviders()
}
// ListProviders returns the names of all registered providers
func (r *ProviderRegistry) ListProviders() []string {
r.mu.RLock()
defer r.mu.RUnlock()
names := make([]string, 0, len(r.providers))
for name := range r.providers {
names = append(names, name)
}
return names
}
// CloseAll closes all provider instances
func CloseAll() error {
return defaultRegistry.CloseAll()
}
// CloseAll closes all provider instances in this registry
func (r *ProviderRegistry) CloseAll() error {
r.mu.Lock()
defer r.mu.Unlock()
var allErrors []error
for name, instance := range r.instances {
if err := instance.Close(); err != nil {
allErrors = append(allErrors, fmt.Errorf("failed to close KMS provider '%s': %w", name, err))
}
}
// Clear the instances map
r.instances = make(map[string]KMSProvider)
return errors.Join(allErrors...)
}
// WithKMSProvider is a helper function to execute code with a KMS provider
func WithKMSProvider(name string, config util.Configuration, fn func(KMSProvider) error) error {
provider, err := GetProvider(name, config)
if err != nil {
return err
}
return fn(provider)
}
// TestKMSConnection tests the connection to a KMS provider
func TestKMSConnection(ctx context.Context, provider KMSProvider, testKeyID string) error {
if provider == nil {
return fmt.Errorf("KMS provider is nil")
}
// Try to describe a test key to verify connectivity
_, err := provider.DescribeKey(ctx, &DescribeKeyRequest{
KeyID: testKeyID,
})
if err != nil {
// If the key doesn't exist, that's still a successful connection test
if kmsErr, ok := err.(*KMSError); ok && kmsErr.Code == ErrCodeNotFoundException {
return nil
}
return fmt.Errorf("KMS connection test failed: %v", err)
}
return nil
}