mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* wdclient: let master-pushed adds replace the gateway's lookup-fed locations After a master leader change the S3 gateway clears its FilerClient cache and refills it through filer lookups. The filer's own master client may still answer from the previous leader's generation until the new leader has heard from the volume servers, so a volume server lost together with the old leader is cached again. The new leader never knew that server and never pushes its removal, and the gateway keeps trying it until reads fail on it volume by volume, one 30 s dial timeout at a time. The gateway's master client relearns every volume from the new leader's adds. On each add, cache the volume where the master client now knows it to be, in place of what a lookup cached, and keep lookups already in flight from caching their answer over it. A provider answer for a volume is now cached whole or not at all. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: take a lookup's fill epoch before its second cache check A master add that landed between the second cache check and the read of the fill epoch was invisible to both: the check had already missed, and the epoch the lookup then read already included the confirmation, so a filer answer that still named the lost server was merged back into the confirmed entry. Taking the epoch first means such an add either shows up in the check or stops the answer from being cached. The same order covers a reset or clear in that gap. TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook at that point and fails with the old order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: gate the master lookup provider's cache writes on the map epoch The provider wrote each answer straight into the vid map, so a lookup sent before a master reset and answered after it landed in the new generation, which a gateway's FilerClient now reads as the current master's view. The answer now goes through fillLocations, which drops it once a reset passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: keep the vidMapClient doc comment on its type Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
885 lines
31 KiB
Go
885 lines
31 KiB
Go
package wdclient
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math/rand"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// UrlPreference controls which URL to use for volume access
|
|
type UrlPreference string
|
|
|
|
const (
|
|
PreferUrl UrlPreference = "url" // Use private URL (default)
|
|
PreferPublicUrl UrlPreference = "publicUrl" // Use public URL
|
|
)
|
|
|
|
// filerHealth tracks the health status of a filer
|
|
type filerHealth struct {
|
|
lastFailureTimeNs int64 // atomic: last failure time in Unix nanoseconds
|
|
failureCount int32 // atomic: consecutive failures
|
|
}
|
|
|
|
// FilerClient provides volume location services by querying a filer
|
|
// It uses the shared vidMap cache for efficient lookups
|
|
// Supports multiple filer addresses with automatic failover for high availability
|
|
// Tracks filer health to avoid repeatedly trying known-unhealthy filers
|
|
// Can discover additional filers from master server when configured with filer group
|
|
type FilerClient struct {
|
|
*vidMapClient
|
|
filerAddresses []pb.ServerAddress
|
|
filerAddressesMu sync.RWMutex // Protects filerAddresses and filerHealth
|
|
filerIndex int32 // atomic: current filer index for round-robin
|
|
filerHealth []*filerHealth // health status per filer (same order as filerAddresses)
|
|
peerUpdates uint64 // pushed filer updates applied; protected by filerAddressesMu
|
|
deferredLeaves map[pb.ServerAddress]struct{} // leaves suppressed while the filer was the last known; protected by filerAddressesMu
|
|
grpcDialOption grpc.DialOption
|
|
urlPreference UrlPreference
|
|
grpcTimeout time.Duration
|
|
cacheSize int // Number of historical vidMap snapshots to keep
|
|
clientId int32 // Unique client identifier for gRPC metadata
|
|
failureThreshold int32 // Circuit breaker: consecutive failures before circuit opens
|
|
resetTimeout time.Duration // Circuit breaker: time before re-checking unhealthy filer
|
|
maxRetries int // Retry: maximum retry attempts for transient failures
|
|
initialRetryWait time.Duration // Retry: initial wait time before first retry
|
|
retryBackoffFactor float64 // Retry: backoff multiplier for wait time
|
|
|
|
// Filer discovery fields
|
|
masterClient *MasterClient // Optional: for discovering filers in the same group
|
|
filerGroup string // Optional: filer group for discovery
|
|
discoveryInterval time.Duration // How often to refresh filer list from master
|
|
stopDiscovery chan struct{} // Signal to stop discovery goroutine
|
|
closeDiscoveryOnce sync.Once // Ensures discovery channel is closed at most once
|
|
}
|
|
|
|
// filerVolumeProvider implements VolumeLocationProvider by querying filer
|
|
// Supports multiple filer addresses with automatic failover
|
|
type filerVolumeProvider struct {
|
|
filerClient *FilerClient
|
|
}
|
|
|
|
// FilerClientOption holds optional configuration for FilerClient
|
|
type FilerClientOption struct {
|
|
GrpcTimeout time.Duration
|
|
UrlPreference UrlPreference
|
|
CacheSize int // Number of historical vidMap snapshots (0 = use default)
|
|
FailureThreshold int32 // Circuit breaker: consecutive failures before skipping filer (0 = use default of 3)
|
|
ResetTimeout time.Duration // Circuit breaker: time before re-checking unhealthy filer (0 = use default of 30s)
|
|
MaxRetries int // Retry: maximum retry attempts for transient failures (0 = use default of 3)
|
|
InitialRetryWait time.Duration // Retry: initial wait time before first retry (0 = use default of 1s)
|
|
RetryBackoffFactor float64 // Retry: backoff multiplier for wait time (0 = use default of 1.5)
|
|
|
|
// Filer discovery options
|
|
MasterClient *MasterClient // Optional: enables filer discovery from master
|
|
FilerGroup string // Optional: filer group name for discovery (required if MasterClient is set)
|
|
DiscoveryInterval time.Duration // Optional: how often to refresh filer list (0 = use default of 5 minutes)
|
|
}
|
|
|
|
// NewFilerClient creates a new client that queries filer(s) for volume locations
|
|
// Supports multiple filer addresses for high availability with automatic failover
|
|
// Uses sensible defaults: 5-second gRPC timeout, PreferUrl, DefaultVidMapCacheSize
|
|
func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, dataCenter string, opts ...*FilerClientOption) *FilerClient {
|
|
if len(filerAddresses) == 0 {
|
|
glog.Fatal("NewFilerClient requires at least one filer address")
|
|
}
|
|
|
|
// Apply defaults
|
|
grpcTimeout := 5 * time.Second
|
|
urlPref := PreferUrl
|
|
cacheSize := DefaultVidMapCacheSize
|
|
failureThreshold := int32(3) // Default: 3 consecutive failures before circuit opens
|
|
resetTimeout := 30 * time.Second // Default: 30 seconds before re-checking unhealthy filer
|
|
maxRetries := 3 // Default: 3 retry attempts for transient failures
|
|
initialRetryWait := time.Second // Default: 1 second initial retry wait
|
|
retryBackoffFactor := 1.5 // Default: 1.5x backoff multiplier
|
|
var masterClient *MasterClient
|
|
var filerGroup string
|
|
discoveryInterval := 5 * time.Minute // Default: refresh every 5 minutes
|
|
|
|
// Override with provided options
|
|
if len(opts) > 0 && opts[0] != nil {
|
|
opt := opts[0]
|
|
if opt.GrpcTimeout > 0 {
|
|
grpcTimeout = opt.GrpcTimeout
|
|
}
|
|
if opt.UrlPreference != "" {
|
|
urlPref = opt.UrlPreference
|
|
}
|
|
if opt.CacheSize > 0 {
|
|
cacheSize = opt.CacheSize
|
|
}
|
|
if opt.FailureThreshold > 0 {
|
|
failureThreshold = opt.FailureThreshold
|
|
}
|
|
if opt.ResetTimeout > 0 {
|
|
resetTimeout = opt.ResetTimeout
|
|
}
|
|
if opt.MaxRetries > 0 {
|
|
maxRetries = opt.MaxRetries
|
|
}
|
|
if opt.InitialRetryWait > 0 {
|
|
initialRetryWait = opt.InitialRetryWait
|
|
}
|
|
if opt.RetryBackoffFactor > 0 {
|
|
retryBackoffFactor = opt.RetryBackoffFactor
|
|
}
|
|
if opt.MasterClient != nil {
|
|
masterClient = opt.MasterClient
|
|
filerGroup = opt.FilerGroup
|
|
if opt.DiscoveryInterval > 0 {
|
|
discoveryInterval = opt.DiscoveryInterval
|
|
}
|
|
}
|
|
}
|
|
|
|
// Initialize health tracking for each filer
|
|
health := make([]*filerHealth, len(filerAddresses))
|
|
for i := range health {
|
|
health[i] = &filerHealth{}
|
|
}
|
|
|
|
fc := &FilerClient{
|
|
filerAddresses: filerAddresses,
|
|
filerIndex: 0,
|
|
filerHealth: health,
|
|
grpcDialOption: grpcDialOption,
|
|
urlPreference: urlPref,
|
|
grpcTimeout: grpcTimeout,
|
|
cacheSize: cacheSize,
|
|
clientId: rand.Int31(), // Random client ID for gRPC metadata tracking
|
|
failureThreshold: failureThreshold,
|
|
resetTimeout: resetTimeout,
|
|
maxRetries: maxRetries,
|
|
initialRetryWait: initialRetryWait,
|
|
retryBackoffFactor: retryBackoffFactor,
|
|
masterClient: masterClient,
|
|
filerGroup: filerGroup,
|
|
discoveryInterval: discoveryInterval,
|
|
}
|
|
|
|
// Start filer discovery if master client is configured
|
|
// Empty filerGroup is valid (represents default group)
|
|
if masterClient != nil {
|
|
fc.stopDiscovery = make(chan struct{})
|
|
go fc.discoverFilers()
|
|
glog.V(0).Infof("FilerClient: started filer discovery for group '%s' (refresh interval: %v)", filerGroup, discoveryInterval)
|
|
}
|
|
|
|
// Create provider that references this FilerClient for failover support
|
|
provider := &filerVolumeProvider{
|
|
filerClient: fc,
|
|
}
|
|
|
|
fc.vidMapClient = newVidMapClient(provider, dataCenter, cacheSize)
|
|
|
|
return fc
|
|
}
|
|
|
|
// GetCurrentFiler returns the currently active filer address
|
|
// This is the filer that was last successfully used or the one indicated by round-robin
|
|
// Returns empty string if no filers are configured
|
|
func (fc *FilerClient) GetCurrentFiler() pb.ServerAddress {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
if len(fc.filerAddresses) == 0 {
|
|
return ""
|
|
}
|
|
|
|
// Get current index (atomically updated on successful operations)
|
|
index := atomic.LoadInt32(&fc.filerIndex)
|
|
if index >= int32(len(fc.filerAddresses)) {
|
|
index = 0
|
|
}
|
|
|
|
return fc.filerAddresses[index]
|
|
}
|
|
|
|
// GetAllFilers returns a snapshot of all filer addresses
|
|
// Returns a copy to avoid concurrent modification issues
|
|
func (fc *FilerClient) GetAllFilers() []pb.ServerAddress {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
// Return a copy to avoid concurrent modification
|
|
filers := make([]pb.ServerAddress, len(fc.filerAddresses))
|
|
copy(filers, fc.filerAddresses)
|
|
return filers
|
|
}
|
|
|
|
// SetCurrentFiler updates the current filer index to the specified address
|
|
// This is useful after successful failover to prefer the healthy filer for future requests
|
|
func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
// Find the index of the specified filer address
|
|
for i, filer := range fc.filerAddresses {
|
|
if filer == addr {
|
|
atomic.StoreInt32(&fc.filerIndex, int32(i))
|
|
return
|
|
}
|
|
}
|
|
// If address not found, leave index unchanged
|
|
}
|
|
|
|
// ShouldSkipUnhealthyFiler checks if a filer address should be skipped based on health tracking
|
|
// Returns true if the filer has exceeded failure threshold and reset timeout hasn't elapsed
|
|
func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
// Find the health for this filer address
|
|
for i, filer := range fc.filerAddresses {
|
|
if filer == addr {
|
|
if i < len(fc.filerHealth) {
|
|
return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i])
|
|
}
|
|
return false
|
|
}
|
|
}
|
|
// If address not found, don't skip it
|
|
return false
|
|
}
|
|
|
|
// RecordFilerSuccess resets failure tracking for a successful filer
|
|
func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
// Find the health for this filer address
|
|
for i, filer := range fc.filerAddresses {
|
|
if filer == addr {
|
|
if i < len(fc.filerHealth) {
|
|
fc.recordFilerSuccessWithHealth(fc.filerHealth[i])
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// RecordFilerFailure increments failure count for an unhealthy filer
|
|
func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
|
|
// Find the health for this filer address
|
|
for i, filer := range fc.filerAddresses {
|
|
if filer == addr {
|
|
if i < len(fc.filerHealth) {
|
|
fc.recordFilerFailureWithHealth(fc.filerHealth[i])
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Close stops the filer discovery goroutine if running
|
|
// Safe to call multiple times (idempotent)
|
|
func (fc *FilerClient) Close() {
|
|
if fc.stopDiscovery != nil {
|
|
fc.closeDiscoveryOnce.Do(func() {
|
|
close(fc.stopDiscovery)
|
|
})
|
|
}
|
|
}
|
|
|
|
// discoverFilers periodically queries the master to discover filers in the same group
|
|
// and updates the filer list. This runs in a background goroutine.
|
|
func (fc *FilerClient) discoverFilers() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
glog.Errorf("FilerClient: panic in filer discovery goroutine for group '%s': %v", fc.filerGroup, r)
|
|
}
|
|
}()
|
|
|
|
// Do an initial discovery
|
|
fc.refreshFilerList()
|
|
|
|
ticker := time.NewTicker(fc.discoveryInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
fc.refreshFilerList()
|
|
case <-fc.stopDiscovery:
|
|
glog.V(0).Infof("FilerClient: stopping filer discovery for group '%s'", fc.filerGroup)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// refreshFilerList queries the master for the current list of filers and updates the local list
|
|
func (fc *FilerClient) refreshFilerList() {
|
|
if fc.masterClient == nil {
|
|
return
|
|
}
|
|
|
|
// Get current master address
|
|
currentMaster := fc.masterClient.GetMaster(context.Background())
|
|
if currentMaster == "" {
|
|
glog.V(1).Infof("FilerClient: no master available for filer discovery")
|
|
return
|
|
}
|
|
|
|
generation := fc.peerUpdateGeneration()
|
|
|
|
// Query master for filers in our group
|
|
updates, err := cluster.ListExistingPeerUpdates(context.Background(), currentMaster, fc.grpcDialOption, fc.filerGroup, cluster.FilerType)
|
|
if err != nil {
|
|
glog.V(1).Infof("FilerClient: failed to list filers in group '%s': %v", fc.filerGroup, err)
|
|
return
|
|
}
|
|
|
|
if len(updates) == 0 {
|
|
glog.V(2).Infof("FilerClient: no filers found in group '%s'", fc.filerGroup)
|
|
return
|
|
}
|
|
|
|
// Build new filer address set
|
|
discoveredFilers := make(map[pb.ServerAddress]struct{}, len(updates))
|
|
for _, update := range updates {
|
|
if update.Address != "" {
|
|
discoveredFilers[pb.ServerAddress(update.Address)] = struct{}{}
|
|
}
|
|
}
|
|
|
|
// Ignore snapshots whose addresses are all empty; reconciling against an
|
|
// empty set would wipe the in-memory list.
|
|
if len(discoveredFilers) == 0 {
|
|
glog.V(1).Infof("FilerClient: discovery snapshot for group '%s' had no usable addresses, keeping existing list", fc.filerGroup)
|
|
return
|
|
}
|
|
|
|
fc.applyDiscoverySnapshot(discoveredFilers, generation)
|
|
}
|
|
|
|
func (fc *FilerClient) peerUpdateGeneration() uint64 {
|
|
fc.filerAddressesMu.RLock()
|
|
defer fc.filerAddressesMu.RUnlock()
|
|
return fc.peerUpdates
|
|
}
|
|
|
|
// applyDiscoverySnapshot drops a snapshot requested before a pushed update was
|
|
// applied: the push is newer, and the next poll reconciles anything it missed.
|
|
func (fc *FilerClient) applyDiscoverySnapshot(discoveredFilers map[pb.ServerAddress]struct{}, generation uint64) {
|
|
fc.filerAddressesMu.Lock()
|
|
defer fc.filerAddressesMu.Unlock()
|
|
if fc.peerUpdates != generation {
|
|
glog.V(1).Infof("FilerClient: discarding discovery snapshot for group '%s' superseded by pushed updates", fc.filerGroup)
|
|
return
|
|
}
|
|
fc.deferredLeaves = nil
|
|
fc.applyDiscoveredFilersLocked(discoveredFilers)
|
|
}
|
|
|
|
// applyDiscoveredFilers treats the master snapshot as authoritative: survivors
|
|
// keep their health counters, new addresses get fresh health, addresses missing
|
|
// from the snapshot are pruned so replaced pods (e.g. rolled K8s filer pods
|
|
// with new IPs) don't linger and get retried after the circuit-breaker reset.
|
|
func (fc *FilerClient) applyDiscoveredFilers(discoveredFilers map[pb.ServerAddress]struct{}) {
|
|
fc.filerAddressesMu.Lock()
|
|
defer fc.filerAddressesMu.Unlock()
|
|
fc.applyDiscoveredFilersLocked(discoveredFilers)
|
|
}
|
|
|
|
// OnPeerUpdate applies a filer join/leave pushed by the master, so the list
|
|
// tracks a rolling restart instead of waiting for the next discovery poll. A
|
|
// leave that would empty the list is deferred until another filer exists to
|
|
// take over: an empty list fails every request, while a stale one still
|
|
// recovers through the poll.
|
|
func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time.Time) {
|
|
if update.NodeType != cluster.FilerType || update.Address == "" {
|
|
return
|
|
}
|
|
addr := pb.ServerAddress(update.Address)
|
|
|
|
fc.filerAddressesMu.Lock()
|
|
defer fc.filerAddressesMu.Unlock()
|
|
|
|
filers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)+1)
|
|
for _, f := range fc.filerAddresses {
|
|
filers[f] = struct{}{}
|
|
}
|
|
|
|
changed := false
|
|
if update.IsAdd {
|
|
// A rejoin cancels its deferred leave; bumping the generation keeps an
|
|
// in-flight snapshot that lacks the rejoined filer from pruning it.
|
|
if _, ok := fc.deferredLeaves[addr]; ok {
|
|
delete(fc.deferredLeaves, addr)
|
|
changed = true
|
|
}
|
|
if _, ok := filers[addr]; !ok {
|
|
filers[addr] = struct{}{}
|
|
changed = true
|
|
}
|
|
} else {
|
|
if _, ok := filers[addr]; ok {
|
|
delete(filers, addr)
|
|
changed = true
|
|
if len(filers) == 0 {
|
|
filers[addr] = struct{}{}
|
|
if fc.deferredLeaves == nil {
|
|
fc.deferredLeaves = make(map[pb.ServerAddress]struct{})
|
|
}
|
|
fc.deferredLeaves[addr] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
// A deferred leave can be honored once another filer exists to take over.
|
|
for gone := range fc.deferredLeaves {
|
|
if _, ok := filers[gone]; ok && len(filers) <= 1 {
|
|
continue
|
|
}
|
|
delete(filers, gone)
|
|
delete(fc.deferredLeaves, gone)
|
|
changed = true
|
|
}
|
|
|
|
if !changed {
|
|
return
|
|
}
|
|
fc.peerUpdates++
|
|
fc.applyDiscoveredFilersLocked(filers)
|
|
}
|
|
|
|
func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.ServerAddress]struct{}) {
|
|
existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses))
|
|
for _, f := range fc.filerAddresses {
|
|
existingFilers[f] = struct{}{}
|
|
}
|
|
|
|
var newFilers []pb.ServerAddress
|
|
for addr := range discoveredFilers {
|
|
if _, found := existingFilers[addr]; !found {
|
|
newFilers = append(newFilers, addr)
|
|
}
|
|
}
|
|
|
|
var removedFilers []pb.ServerAddress
|
|
for _, f := range fc.filerAddresses {
|
|
if _, found := discoveredFilers[f]; !found {
|
|
removedFilers = append(removedFilers, f)
|
|
}
|
|
}
|
|
|
|
if len(newFilers) == 0 && len(removedFilers) == 0 {
|
|
return
|
|
}
|
|
|
|
// Remember the active filer so the round-robin pointer can follow it across the rebuild.
|
|
currentIndex := atomic.LoadInt32(&fc.filerIndex)
|
|
var currentFiler pb.ServerAddress
|
|
if currentIndex >= 0 && currentIndex < int32(len(fc.filerAddresses)) {
|
|
currentFiler = fc.filerAddresses[currentIndex]
|
|
}
|
|
|
|
newAddresses := make([]pb.ServerAddress, 0, len(fc.filerAddresses)-len(removedFilers)+len(newFilers))
|
|
newHealth := make([]*filerHealth, 0, cap(newAddresses))
|
|
for i, f := range fc.filerAddresses {
|
|
if _, found := discoveredFilers[f]; found {
|
|
newAddresses = append(newAddresses, f)
|
|
newHealth = append(newHealth, fc.filerHealth[i])
|
|
}
|
|
}
|
|
for _, f := range newFilers {
|
|
newAddresses = append(newAddresses, f)
|
|
newHealth = append(newHealth, &filerHealth{})
|
|
}
|
|
|
|
fc.filerAddresses = newAddresses
|
|
fc.filerHealth = newHealth
|
|
|
|
var newIndex int32
|
|
if currentFiler != "" {
|
|
for i, f := range newAddresses {
|
|
if f == currentFiler {
|
|
newIndex = int32(i)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
atomic.StoreInt32(&fc.filerIndex, newIndex)
|
|
|
|
if len(removedFilers) > 0 {
|
|
glog.V(0).Infof("FilerClient: removed %d filer(s) no longer in group '%s': %v", len(removedFilers), fc.filerGroup, removedFilers)
|
|
}
|
|
if len(newFilers) > 0 {
|
|
glog.V(0).Infof("FilerClient: discovered %d new filer(s) in group '%s': %v", len(newFilers), fc.filerGroup, newFilers)
|
|
}
|
|
}
|
|
|
|
// OnVolumeLocationChange applies master-pushed changes to the lookup-fed cache,
|
|
// so a retired server stops being tried before a read against it fails. An
|
|
// add lifts the removal fence and caches the volume where the master client
|
|
// now knows it to be, replacing what a lookup cached for it.
|
|
//
|
|
// The replacement matters after a leader change. The reconnect clears this
|
|
// cache, and lookups refill it from the filer, whose own master client can
|
|
// still answer from the previous leader's view until the new leader has heard
|
|
// from the volume servers. A server lost with the old leader is then cached
|
|
// again, and the new leader, which never knew it, never pushes its removal.
|
|
// Its adds for the volumes that server held are what correct the refill.
|
|
func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVids []uint32) {
|
|
for _, vid := range deletedVids {
|
|
fc.tombstone(vid, loc)
|
|
fc.deleteLocation(vid, loc)
|
|
}
|
|
for _, vid := range addedVids {
|
|
fc.untombstone(vid, loc)
|
|
if fc.masterClient == nil {
|
|
continue
|
|
}
|
|
if locations, found := fc.masterClient.vidMap.currentLocations(vid); found {
|
|
fc.confirmLocations(vid, locations)
|
|
}
|
|
}
|
|
}
|
|
|
|
// OnVidMapReset drops the cache, since removals made while the master link
|
|
// was down are never pushed.
|
|
func (fc *FilerClient) OnVidMapReset() {
|
|
fc.clearVidMap()
|
|
}
|
|
|
|
// GetLookupFileIdFunction returns a lookup function with URL preference handling
|
|
func (fc *FilerClient) GetLookupFileIdFunction() LookupFileIdFunctionType {
|
|
if fc.urlPreference == PreferUrl {
|
|
// Use the default implementation from vidMapClient
|
|
return fc.vidMapClient.GetLookupFileIdFunction()
|
|
}
|
|
|
|
// Custom implementation that prefers PublicUrl
|
|
return func(ctx context.Context, fileId string) (fullUrls []string, err error) {
|
|
// Parse file ID to extract volume ID
|
|
parts := strings.Split(fileId, ",")
|
|
if len(parts) != 2 {
|
|
return nil, fmt.Errorf("invalid fileId format: %s", fileId)
|
|
}
|
|
volumeIdStr := parts[0]
|
|
|
|
// First try the cache using LookupVolumeIdsWithFallback
|
|
vidLocations, err := fc.LookupVolumeIdsWithFallback(ctx, []string{volumeIdStr})
|
|
|
|
// Check for partial results first (important for multi-volume batched lookups)
|
|
locations, found := vidLocations[volumeIdStr]
|
|
if !found || len(locations) == 0 {
|
|
// Volume not found - return specific error with context from lookup if available
|
|
if err != nil {
|
|
return nil, fmt.Errorf("volume %s not found for fileId %s: %w", volumeIdStr, fileId, err)
|
|
}
|
|
return nil, fmt.Errorf("volume %s not found for fileId %s", volumeIdStr, fileId)
|
|
}
|
|
|
|
// Volume found successfully - ignore any errors about other volumes
|
|
// (not relevant for single-volume lookup, but defensive for future batching)
|
|
|
|
// Build URLs with publicUrl preference, and also prefer same DC
|
|
var sameDcUrls, otherDcUrls []string
|
|
localUrls := make(map[string]bool)
|
|
dataCenter := fc.GetDataCenter()
|
|
for _, loc := range locations {
|
|
url := loc.PublicUrl
|
|
if url == "" {
|
|
url = loc.Url
|
|
}
|
|
httpUrl := "http://" + url + "/" + fileId
|
|
glog.V(4).Infof("lookup %s => %s, data in remote storage tier: %v", fileId, url, loc.DataInRemote)
|
|
if !loc.DataInRemote {
|
|
localUrls[httpUrl] = true
|
|
}
|
|
if dataCenter != "" && dataCenter == loc.DataCenter {
|
|
sameDcUrls = append(sameDcUrls, httpUrl)
|
|
} else {
|
|
otherDcUrls = append(otherDcUrls, httpUrl)
|
|
}
|
|
}
|
|
// Shuffle to distribute load across volume servers
|
|
rand.Shuffle(len(sameDcUrls), func(i, j int) { sameDcUrls[i], sameDcUrls[j] = sameDcUrls[j], sameDcUrls[i] })
|
|
rand.Shuffle(len(otherDcUrls), func(i, j int) { otherDcUrls[i], otherDcUrls[j] = otherDcUrls[j], otherDcUrls[i] })
|
|
// Local replicas go first inside each data center, but never ahead of
|
|
// the data-center preference itself. Mirrors
|
|
// vidMap.LookupVolumeServerUrl so all client lookup paths agree.
|
|
if len(localUrls) > 0 {
|
|
sameDcUrls = util.ReorderToFront(localUrls, sameDcUrls)
|
|
otherDcUrls = util.ReorderToFront(localUrls, otherDcUrls)
|
|
}
|
|
fullUrls = append(sameDcUrls, otherDcUrls...)
|
|
return fullUrls, nil
|
|
}
|
|
}
|
|
|
|
// isRetryableGrpcError checks if a gRPC error is transient and should be retried
|
|
//
|
|
// Note on codes.Aborted: While Aborted can indicate application-level conflicts
|
|
// (e.g., transaction failures), in the context of volume location lookups (which
|
|
// are simple read-only operations with no transactions), Aborted is more likely
|
|
// to indicate transient server issues during restart/recovery. We include it here
|
|
// for volume lookups but log it for visibility in case misclassification occurs.
|
|
func isRetryableGrpcError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
|
|
// Check gRPC status code
|
|
st, ok := status.FromError(err)
|
|
if ok {
|
|
switch st.Code() {
|
|
case codes.Unavailable: // Server unavailable (temporary)
|
|
return true
|
|
case codes.DeadlineExceeded: // Request timeout
|
|
return true
|
|
case codes.ResourceExhausted: // Rate limited or overloaded
|
|
return true
|
|
case codes.Aborted:
|
|
// Aborted during read-only volume lookups is likely transient
|
|
// (e.g., filer restarting), but log for visibility
|
|
glog.V(1).Infof("Treating Aborted as retryable for volume lookup: %v", err)
|
|
return true
|
|
}
|
|
}
|
|
|
|
// Fallback for non-gRPC errors (e.g. network errors). "connection" and
|
|
// "timeout" are deliberately broader than the shared classifier: a volume
|
|
// lookup is a cheap read-only call, so leaning towards a retry is fine.
|
|
errStr := strings.ToLower(err.Error())
|
|
return util.IsTransientError(err) ||
|
|
strings.Contains(errStr, "connection") ||
|
|
strings.Contains(errStr, "timeout")
|
|
}
|
|
|
|
// jitter returns a duration in the range [d/2, d) using equal jitter.
|
|
// This prevents thundering herds when many clients retry simultaneously
|
|
// after a transient failure (e.g., network partition healing).
|
|
func jitter(d time.Duration) time.Duration {
|
|
if d <= 0 {
|
|
return 0
|
|
}
|
|
half := d / 2
|
|
if half <= 0 {
|
|
return d
|
|
}
|
|
return half + time.Duration(rand.Int63n(int64(half)))
|
|
}
|
|
|
|
// shouldSkipUnhealthyFiler checks if we should skip a filer based on recent failures
|
|
// Circuit breaker pattern: skip filers with multiple recent consecutive failures
|
|
// shouldSkipUnhealthyFilerWithHealth checks if a filer should be skipped based on health
|
|
// Uses atomic operations only - safe to call without locks
|
|
func (fc *FilerClient) shouldSkipUnhealthyFilerWithHealth(health *filerHealth) bool {
|
|
failureCount := atomic.LoadInt32(&health.failureCount)
|
|
|
|
// Check if failure count exceeds threshold
|
|
if failureCount < fc.failureThreshold {
|
|
return false
|
|
}
|
|
|
|
// Re-check unhealthy filers after reset timeout
|
|
lastFailureNs := atomic.LoadInt64(&health.lastFailureTimeNs)
|
|
if lastFailureNs == 0 {
|
|
return false // Never failed, shouldn't skip
|
|
}
|
|
lastFailureTime := time.Unix(0, lastFailureNs)
|
|
if time.Since(lastFailureTime) > fc.resetTimeout {
|
|
return false // Time to re-check
|
|
}
|
|
|
|
return true // Skip this unhealthy filer
|
|
}
|
|
|
|
// recordFilerSuccessWithHealth resets failure tracking for a successful filer
|
|
func (fc *FilerClient) recordFilerSuccessWithHealth(health *filerHealth) {
|
|
atomic.StoreInt32(&health.failureCount, 0)
|
|
}
|
|
|
|
// recordFilerFailureWithHealth increments failure count for an unhealthy filer
|
|
func (fc *FilerClient) recordFilerFailureWithHealth(health *filerHealth) {
|
|
atomic.AddInt32(&health.failureCount, 1)
|
|
atomic.StoreInt64(&health.lastFailureTimeNs, time.Now().UnixNano())
|
|
}
|
|
|
|
// LookupVolumeIds queries the filer for volume locations with automatic failover
|
|
// Tries all configured filer addresses until one succeeds (high availability)
|
|
// Retries transient gRPC errors (Unavailable, DeadlineExceeded, etc.) with exponential backoff
|
|
// Note: Unlike master's VolumeIdLocation, filer's Locations message doesn't currently have
|
|
// an Error field. This implementation handles the current structure while being prepared
|
|
// for future error reporting enhancements.
|
|
func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
|
|
fc := p.filerClient
|
|
result := make(map[string][]Location)
|
|
|
|
// Retry transient failures with configurable backoff
|
|
var lastErr error
|
|
waitTime := fc.initialRetryWait
|
|
maxRetries := fc.maxRetries
|
|
|
|
for retry := 0; retry < maxRetries; retry++ {
|
|
// Try all filer addresses with round-robin starting from current index
|
|
// Skip known-unhealthy filers (circuit breaker pattern)
|
|
i := atomic.LoadInt32(&fc.filerIndex)
|
|
|
|
// Get filer count with read lock
|
|
fc.filerAddressesMu.RLock()
|
|
n := int32(len(fc.filerAddresses))
|
|
fc.filerAddressesMu.RUnlock()
|
|
|
|
for x := int32(0); x < n; x++ {
|
|
// Get current filer address and health with read lock
|
|
fc.filerAddressesMu.RLock()
|
|
if len(fc.filerAddresses) == 0 {
|
|
fc.filerAddressesMu.RUnlock()
|
|
lastErr = fmt.Errorf("no filers available")
|
|
break
|
|
}
|
|
if i >= int32(len(fc.filerAddresses)) {
|
|
// Filer list changed, reset index
|
|
i = 0
|
|
}
|
|
|
|
// Get health pointer while holding lock
|
|
health := fc.filerHealth[i]
|
|
filerAddress := fc.filerAddresses[i]
|
|
fc.filerAddressesMu.RUnlock()
|
|
|
|
// Circuit breaker: skip unhealthy filers (no lock needed - uses atomics)
|
|
if fc.shouldSkipUnhealthyFilerWithHealth(health) {
|
|
glog.V(2).Infof("FilerClient: skipping unhealthy filer %s (consecutive failures: %d)",
|
|
filerAddress, atomic.LoadInt32(&health.failureCount))
|
|
i++
|
|
if i >= n {
|
|
i = 0
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Use anonymous function to ensure defer cancel() is called per iteration, not accumulated
|
|
err := func() error {
|
|
// Create a fresh timeout context for each filer attempt
|
|
// This ensures each retry gets the full grpcTimeout, not a diminishing deadline
|
|
timeoutCtx, cancel := context.WithTimeout(ctx, fc.grpcTimeout)
|
|
defer cancel() // Always clean up context, even on panic or early return
|
|
|
|
return pb.WithGrpcFilerClient(false, fc.clientId, filerAddress, fc.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
|
resp, err := client.LookupVolume(timeoutCtx, &filer_pb.LookupVolumeRequest{
|
|
VolumeIds: volumeIds,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("filer.LookupVolume failed: %w", err)
|
|
}
|
|
|
|
// Process each volume in the response
|
|
for vid, locs := range resp.LocationsMap {
|
|
// Convert locations from protobuf to internal format
|
|
var locations []Location
|
|
for _, loc := range locs.Locations {
|
|
locations = append(locations, Location{
|
|
Url: loc.Url,
|
|
PublicUrl: loc.PublicUrl,
|
|
DataCenter: loc.DataCenter,
|
|
GrpcPort: int(loc.GrpcPort),
|
|
})
|
|
}
|
|
|
|
// Only add to result if we have locations
|
|
// Empty locations with no gRPC error means "not found" (volume doesn't exist)
|
|
if len(locations) > 0 {
|
|
result[vid] = locations
|
|
glog.V(4).Infof("FilerClient: volume %s found with %d location(s)", vid, len(locations))
|
|
} else {
|
|
glog.V(2).Infof("FilerClient: volume %s not found (no locations in response)", vid)
|
|
}
|
|
}
|
|
|
|
// Check for volumes that weren't in the response at all
|
|
// This could indicate a problem with the filer
|
|
for _, vid := range volumeIds {
|
|
if _, found := resp.LocationsMap[vid]; !found {
|
|
glog.V(1).Infof("FilerClient: volume %s missing from filer response", vid)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}()
|
|
|
|
if err != nil {
|
|
// a caller that gave up says nothing about this filer; gRPC can
|
|
// surface its own DeadlineExceeded a tick before ctx.Err()
|
|
// becomes visible, so reinterpret only deadline errors this way
|
|
ctxErr := ctx.Err()
|
|
if ctxErr == nil && isDeadlineExceededErr(err) {
|
|
if dl, ok := ctx.Deadline(); ok && !dl.After(time.Now()) {
|
|
ctxErr = context.DeadlineExceeded
|
|
}
|
|
}
|
|
if ctxErr != nil {
|
|
if lastErr != nil {
|
|
return nil, fmt.Errorf("%w (earlier filer error: %v)", ctxErr, lastErr)
|
|
}
|
|
return nil, ctxErr
|
|
}
|
|
glog.V(1).Infof("FilerClient: filer %s lookup failed (attempt %d/%d, retry %d/%d): %v", filerAddress, x+1, n, retry+1, maxRetries, err)
|
|
fc.recordFilerFailureWithHealth(health)
|
|
lastErr = err
|
|
i++
|
|
if i >= n {
|
|
i = 0
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Success - update the preferred filer index and reset health tracking
|
|
atomic.StoreInt32(&fc.filerIndex, i)
|
|
fc.recordFilerSuccessWithHealth(health)
|
|
glog.V(3).Infof("FilerClient: looked up %d volumes on %s, found %d", len(volumeIds), filerAddress, len(result))
|
|
return result, nil
|
|
}
|
|
|
|
// All filers failed on this attempt
|
|
// Check if the error is retryable (transient gRPC error)
|
|
if !isRetryableGrpcError(lastErr) {
|
|
// Non-retryable error (e.g., NotFound, PermissionDenied) - fail immediately
|
|
return nil, fmt.Errorf("all %d filer(s) failed with non-retryable error: %w", n, lastErr)
|
|
}
|
|
|
|
// Transient error - retry if we have attempts left
|
|
if retry < maxRetries-1 {
|
|
jitteredWait := jitter(waitTime)
|
|
glog.V(1).Infof("FilerClient: all %d filer(s) failed with retryable error (attempt %d/%d), retrying in %v: %v",
|
|
n, retry+1, maxRetries, jitteredWait, lastErr)
|
|
timer := time.NewTimer(jitteredWait)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return nil, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
waitTime = time.Duration(float64(waitTime) * fc.retryBackoffFactor)
|
|
}
|
|
}
|
|
|
|
// All retries exhausted
|
|
fc.filerAddressesMu.RLock()
|
|
totalFilers := len(fc.filerAddresses)
|
|
fc.filerAddressesMu.RUnlock()
|
|
return nil, fmt.Errorf("all %d filer(s) failed after %d attempts, last error: %w", totalFilers, maxRetries, lastErr)
|
|
}
|