wdclient: let master-pushed adds replace the gateway's lookup-fed locations (#12705)

* wdclient: let master-pushed adds replace the gateway's lookup-fed locations

After a master leader change the S3 gateway clears its FilerClient cache and
refills it through filer lookups. The filer's own master client may still
answer from the previous leader's generation until the new leader has heard
from the volume servers, so a volume server lost together with the old leader
is cached again. The new leader never knew that server and never pushes its
removal, and the gateway keeps trying it until reads fail on it volume by
volume, one 30 s dial timeout at a time.

The gateway's master client relearns every volume from the new leader's
adds. On each add, cache the volume where the master client now knows it to
be, in place of what a lookup cached, and keep lookups already in flight from
caching their answer over it. A provider answer for a volume is now cached
whole or not at all.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: take a lookup's fill epoch before its second cache check

A master add that landed between the second cache check and the read of
the fill epoch was invisible to both: the check had already missed, and
the epoch the lookup then read already included the confirmation, so a
filer answer that still named the lost server was merged back into the
confirmed entry. Taking the epoch first means such an add either shows
up in the check or stops the answer from being cached. The same order
covers a reset or clear in that gap.

TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook
at that point and fails with the old order.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: gate the master lookup provider's cache writes on the map epoch

The provider wrote each answer straight into the vid map, so a lookup sent
before a master reset and answered after it landed in the new generation,
which a gateway's FilerClient now reads as the current master's view. The
answer now goes through fillLocations, which drops it once a reset passed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: keep the vidMapClient doc comment on its type

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-17 19:10:20 -08:00
1 parent 91eaabf371
commit 97f49571a1
6 files changed
+178 -22

No files matched your search

+16 -2
View File
@@ -528,8 +528,16 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve
}
// OnVolumeLocationChange applies master-pushed changes to the lookup-fed cache,
// so a retired server stops being tried before a read against it fails. Adds
// only lift the removal fence; locations are still learned by lookup.
// so a retired server stops being tried before a read against it fails. An
// add lifts the removal fence and caches the volume where the master client
// now knows it to be, replacing what a lookup cached for it.
//
// The replacement matters after a leader change. The reconnect clears this
// cache, and lookups refill it from the filer, whose own master client can
// still answer from the previous leader's view until the new leader has heard
// from the volume servers. A server lost with the old leader is then cached
// again, and the new leader, which never knew it, never pushes its removal.
// Its adds for the volumes that server held are what correct the refill.
func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVids []uint32) {
for _, vid := range deletedVids {
fc.tombstone(vid, loc)
@@ -537,6 +545,12 @@ func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVi
}
for _, vid := range addedVids {
fc.untombstone(vid, loc)
if fc.masterClient == nil {
continue
}
if locations, found := fc.masterClient.vidMap.currentLocations(vid); found {
fc.confirmLocations(vid, locations)
}
}
}
+48
View File
@@ -561,3 +561,51 @@ func TestReconnectReplayLiftsFence(t *testing.T) {
expectServed(t, fc, "5", a)
}
// A gateway that reconnects to a new leader drops its cache and refills it
// from a filer that may still answer with the old leader's view, a server
// that died with it included. The new leader never knew that server, so it
// never pushes its removal; its re-told adds are what replace the refill.
func TestLeaderChangeAddsReplaceStaleRefill(t *testing.T) {
dead := Location{Url: "dead:8080"}
live := Location{Url: "live:8080"}
other := Location{Url: "other:8080"}
provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}}
mc := moveClient()
fc := hookedFilerClient(mc, provider)
fc.masterClient = mc
pushVolumeLocation(mc, dead.Url, []uint32{5}, nil, nil, nil)
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
mc.resetVidMap()
expectServed(t, fc, "5", dead, live)
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
expectServed(t, fc, "5", live)
pushVolumeLocation(mc, other.Url, []uint32{5}, nil, nil, nil)
expectServed(t, fc, "5", live, other)
}
// A master add that lands between a lookup's second cache check and its query
// to a filer that has not caught up must not be undone by that filer's answer.
func TestConfirmDuringLookupIsNotUndone(t *testing.T) {
dead := Location{Url: "dead:8080"}
live := Location{Url: "live:8080"}
provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}}
mc := moveClient()
fc := hookedFilerClient(mc, provider)
fc.masterClient = mc
mc.resetVidMap()
afterLookupRecheck = func() {
afterLookupRecheck = nil
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
}
t.Cleanup(func() { afterLookupRecheck = nil })
if _, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{"5"}); err != nil {
t.Fatalf("lookup: %v", err)
}
expectServed(t, fc, "5", live)
}
+2 -2
View File
@@ -72,6 +72,7 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []
func() error {
result = make(map[string][]Location)
lookupErrors = nil
epoch := p.masterClient.mapEpoch()
// Per-attempt timeout bounds both master resolution and the RPC
// so a single attempt cannot consume the entire retry budget.
@@ -124,10 +125,9 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []
ReadOnly: masterLoc.ReadOnly,
ReadOnlyCanDelete: masterLoc.ReadOnlyCanDelete,
}
// Update cache with the location
p.masterClient.addLocation(uint32(vid), loc)
locations = append(locations, loc)
}
p.masterClient.fillLocations(uint32(vid), locations, epoch)
if len(locations) > 0 {
result[vidOnly] = locations
+37
View File
@@ -117,3 +117,40 @@ func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) {
t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed)
}
}
type resetDuringLookupServer struct {
fakeLookupServer
mc *MasterClient
}
func (s *resetDuringLookupServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
s.mc.resetVidMap()
return s.fakeLookupServer.LookupVolume(ctx, req)
}
// An answer sent before a reset must not land in the new generation, which a
// gateway's FilerClient reads as the current master's view.
func TestLookupAnswerCrossingResetIsNotCached(t *testing.T) {
mc := NewMasterClient(
grpc.WithTransportCredentials(insecure.NewCredentials()),
"", "test", "", "", "",
pb.ServerDiscovery{},
)
srv := &resetDuringLookupServer{mc: mc}
mc.setCurrentMaster(startFakeMasterServer(t, srv))
mc.grpcTimeout = 5 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
provider := &masterVolumeProvider{masterClient: mc}
result, err := provider.LookupVolumeIds(ctx, []string{"1"})
if err != nil {
t.Fatalf("lookup: %v", err)
}
if _, ok := result["1"]; !ok {
t.Fatal("expected volume 1 to be served")
}
if locations, found := mc.vidMap.currentLocations(1); found {
t.Fatalf("answer that crossed a reset was cached: %v", locations)
}
}
+35
View File
@@ -225,6 +225,41 @@ func (vc *vidMap) hasLocation(vid uint32, url string) bool {
return false
}
// currentLocations returns what vid was learned at since the last reset,
// leaving out anything only an earlier generation reported. When both a
// regular and an ec entry are current, the regular copies serve, as in
// GetLocations.
func (vc *vidMap) currentLocations(vid uint32) ([]Location, bool) {
vc.RLock()
defer vc.RUnlock()
for _, vid2Locations := range []map[uint32]*locationsEntry{vc.vid2Locations, vc.ecVid2Locations} {
if entry, found := lookupEntry(vid2Locations, vid); found && entry.generation == vc.generation {
return entry.locations, true
}
}
return nil, false
}
// setLocations makes locations vid's regular entry in the current
// generation, in place of whatever the entry held.
func (vc *vidMap) setLocations(vid uint32, locations []Location) {
vc.Lock()
defer vc.Unlock()
if entry, found := vc.vid2Locations[vid]; found {
vc.releaseEntry(entry)
}
if len(locations) == 0 {
delete(vc.vid2Locations, vid)
return
}
stored := make([]Location, len(locations))
copy(stored, locations)
vc.vid2Locations[vid] = &locationsEntry{locations: stored, generation: vc.generation}
for _, loc := range stored {
vc.incrementServerRef(locationServerKey(loc))
}
}
// lookupEntry returns vid's entry when it still holds locations. Callers must
// hold the lock.
func lookupEntry(vid2Locations map[uint32]*locationsEntry, vid uint32) (*locationsEntry, bool) {
+40 -18
View File
@@ -27,6 +27,10 @@ type VolumeLocationProvider interface {
LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error)
}
// afterLookupRecheck, when set by a test, runs between a lookup's second cache
// check and its provider query.
var afterLookupRecheck func()
// vidMapClient provides volume location caching with pluggable lookup
// It wraps the battle-tested vidMap with customizable volume lookup strategies
type vidMapClient struct {
@@ -210,6 +214,11 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI
batchKey := strings.Join(needsLookup, ",")
sfResult, err := vc.doSharedLookup(ctx, batchKey, func(ctx context.Context) (interface{}, error) {
// The epoch is taken before the cache is checked again: a confirmation
// or reset that lands after it either shows up in that check or stops
// the provider's answer from being cached.
fillEpoch := vc.mapEpoch()
// Double-check cache for volumes that might have been populated while waiting
stillNeedLookup := make([]string, 0, len(needsLookup))
batchResult := make(map[string][]Location)
@@ -223,6 +232,10 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI
}
}
if afterLookupRecheck != nil {
afterLookupRecheck()
}
if len(stillNeedLookup) == 0 {
return batchResult, nil
}
@@ -230,7 +243,6 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI
// Query provider with batched volume IDs
glog.V(2).Infof("Looking up %d volumes from provider: %v", len(stillNeedLookup), stillNeedLookup)
fillEpoch := vc.mapEpoch()
providerResults, err := vc.provider.LookupVolumeIds(ctx, stillNeedLookup)
if err != nil {
return batchResult, fmt.Errorf("provider lookup failed: %w", err)
@@ -243,13 +255,7 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI
continue
}
var live []Location
for _, loc := range locations {
if vc.fillLocation(uint32(vid), loc, fillEpoch) {
live = append(live, loc)
}
}
live := vc.fillLocations(uint32(vid), locations, fillEpoch)
if len(live) > 0 {
batchResult[vidString] = live
}
@@ -438,20 +444,36 @@ func (vc *vidMapClient) untombstone(vid uint32, loc Location) {
}
}
// fillLocation reports whether a provider answer may be served, and caches it
// unless a reset or clear happened since epoch. Holding recentDeletesMu across
// the check and the add keeps a concurrent removal from being undone.
func (vc *vidMapClient) fillLocation(vid uint32, loc Location, epoch uint64) bool {
// fillLocations returns the locations of a provider answer for vid that may
// be served, and caches them unless a reset, clear or confirmation happened
// since epoch. Holding recentDeletesMu across the check and the add keeps a
// concurrent removal or confirmation from being undone, and caches the
// answer for vid whole or not at all.
func (vc *vidMapClient) fillLocations(vid uint32, locations []Location, epoch uint64) (live []Location) {
vc.recentDeletesMu.Lock()
defer vc.recentDeletesMu.Unlock()
vc.expireRecentDeletes(time.Now())
if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found {
return false
for _, loc := range locations {
if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found {
continue
}
live = append(live, loc)
if vc.vidMapEpoch == epoch {
vc.vidMap.addLocation(vid, loc)
}
}
if vc.vidMapEpoch == epoch {
vc.vidMap.addLocation(vid, loc)
}
return true
return live
}
// confirmLocations caches what a master stream reports vid at now in place of
// what lookups cached for it, which may have come from a source that had not
// caught up with the master yet. Lookups already in flight answer from that
// same source, so the epoch moves on and they do not cache what they fetched.
func (vc *vidMapClient) confirmLocations(vid uint32, locations []Location) {
vc.recentDeletesMu.Lock()
defer vc.recentDeletesMu.Unlock()
vc.vidMapEpoch++
vc.vidMap.setLocations(vid, locations)
}
// InvalidateCache removes all cached locations for a volume ID