From 97f49571a1113e7a40257e5cfc42731792b3a026 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Wed, 18 Nov 2026 04:10:20 +0100 Subject: [PATCH] wdclient: let master-pushed adds replace the gateway's lookup-fed locations (#12705) * wdclient: let master-pushed adds replace the gateway's lookup-fed locations After a master leader change the S3 gateway clears its FilerClient cache and refills it through filer lookups. The filer's own master client may still answer from the previous leader's generation until the new leader has heard from the volume servers, so a volume server lost together with the old leader is cached again. The new leader never knew that server and never pushes its removal, and the gateway keeps trying it until reads fail on it volume by volume, one 30 s dial timeout at a time. The gateway's master client relearns every volume from the new leader's adds. On each add, cache the volume where the master client now knows it to be, in place of what a lookup cached, and keep lookups already in flight from caching their answer over it. A provider answer for a volume is now cached whole or not at all. Co-Authored-By: Claude Opus 5.5 * wdclient: take a lookup's fill epoch before its second cache check A master add that landed between the second cache check and the read of the fill epoch was invisible to both: the check had already missed, and the epoch the lookup then read already included the confirmation, so a filer answer that still named the lost server was merged back into the confirmed entry. Taking the epoch first means such an add either shows up in the check or stops the answer from being cached. The same order covers a reset or clear in that gap. TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook at that point and fails with the old order. Co-Authored-By: Claude Opus 5.5 * wdclient: gate the master lookup provider's cache writes on the map epoch The provider wrote each answer straight into the vid map, so a lookup sent before a master reset and answered after it landed in the new generation, which a gateway's FilerClient now reads as the current master's view. The answer now goes through fillLocations, which drops it once a reset passed. Co-Authored-By: Claude Opus 5.5 * wdclient: keep the vidMapClient doc comment on its type Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- weed/wdclient/filer_client.go | 18 +++++++- weed/wdclient/filer_client_test.go | 48 ++++++++++++++++++++ weed/wdclient/masterclient.go | 4 +- weed/wdclient/masterclient_retry_test.go | 37 +++++++++++++++ weed/wdclient/vid_map.go | 35 ++++++++++++++ weed/wdclient/vidmap_client.go | 58 ++++++++++++++++-------- 6 files changed, 178 insertions(+), 22 deletions(-) diff --git a/weed/wdclient/filer_client.go b/weed/wdclient/filer_client.go index cf32328f0c..7f6f8df94f 100644 --- a/weed/wdclient/filer_client.go +++ b/weed/wdclient/filer_client.go @@ -528,8 +528,16 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve } // OnVolumeLocationChange applies master-pushed changes to the lookup-fed cache, -// so a retired server stops being tried before a read against it fails. Adds -// only lift the removal fence; locations are still learned by lookup. +// so a retired server stops being tried before a read against it fails. An +// add lifts the removal fence and caches the volume where the master client +// now knows it to be, replacing what a lookup cached for it. +// +// The replacement matters after a leader change. The reconnect clears this +// cache, and lookups refill it from the filer, whose own master client can +// still answer from the previous leader's view until the new leader has heard +// from the volume servers. A server lost with the old leader is then cached +// again, and the new leader, which never knew it, never pushes its removal. +// Its adds for the volumes that server held are what correct the refill. func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVids []uint32) { for _, vid := range deletedVids { fc.tombstone(vid, loc) @@ -537,6 +545,12 @@ func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVi } for _, vid := range addedVids { fc.untombstone(vid, loc) + if fc.masterClient == nil { + continue + } + if locations, found := fc.masterClient.vidMap.currentLocations(vid); found { + fc.confirmLocations(vid, locations) + } } } diff --git a/weed/wdclient/filer_client_test.go b/weed/wdclient/filer_client_test.go index 97e23dd9cb..a66d88efa2 100644 --- a/weed/wdclient/filer_client_test.go +++ b/weed/wdclient/filer_client_test.go @@ -561,3 +561,51 @@ func TestReconnectReplayLiftsFence(t *testing.T) { expectServed(t, fc, "5", a) } + +// A gateway that reconnects to a new leader drops its cache and refills it +// from a filer that may still answer with the old leader's view, a server +// that died with it included. The new leader never knew that server, so it +// never pushes its removal; its re-told adds are what replace the refill. +func TestLeaderChangeAddsReplaceStaleRefill(t *testing.T) { + dead := Location{Url: "dead:8080"} + live := Location{Url: "live:8080"} + other := Location{Url: "other:8080"} + provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}} + mc := moveClient() + fc := hookedFilerClient(mc, provider) + fc.masterClient = mc + pushVolumeLocation(mc, dead.Url, []uint32{5}, nil, nil, nil) + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + + mc.resetVidMap() + expectServed(t, fc, "5", dead, live) + + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + expectServed(t, fc, "5", live) + + pushVolumeLocation(mc, other.Url, []uint32{5}, nil, nil, nil) + expectServed(t, fc, "5", live, other) +} + +// A master add that lands between a lookup's second cache check and its query +// to a filer that has not caught up must not be undone by that filer's answer. +func TestConfirmDuringLookupIsNotUndone(t *testing.T) { + dead := Location{Url: "dead:8080"} + live := Location{Url: "live:8080"} + provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}} + mc := moveClient() + fc := hookedFilerClient(mc, provider) + fc.masterClient = mc + mc.resetVidMap() + + afterLookupRecheck = func() { + afterLookupRecheck = nil + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + } + t.Cleanup(func() { afterLookupRecheck = nil }) + if _, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{"5"}); err != nil { + t.Fatalf("lookup: %v", err) + } + + expectServed(t, fc, "5", live) +} diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index e738c38871..6cc97f76fc 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -72,6 +72,7 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds [] func() error { result = make(map[string][]Location) lookupErrors = nil + epoch := p.masterClient.mapEpoch() // Per-attempt timeout bounds both master resolution and the RPC // so a single attempt cannot consume the entire retry budget. @@ -124,10 +125,9 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds [] ReadOnly: masterLoc.ReadOnly, ReadOnlyCanDelete: masterLoc.ReadOnlyCanDelete, } - // Update cache with the location - p.masterClient.addLocation(uint32(vid), loc) locations = append(locations, loc) } + p.masterClient.fillLocations(uint32(vid), locations, epoch) if len(locations) > 0 { result[vidOnly] = locations diff --git a/weed/wdclient/masterclient_retry_test.go b/weed/wdclient/masterclient_retry_test.go index 1b24b78c7c..8499858d53 100644 --- a/weed/wdclient/masterclient_retry_test.go +++ b/weed/wdclient/masterclient_retry_test.go @@ -117,3 +117,40 @@ func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) { t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed) } } + +type resetDuringLookupServer struct { + fakeLookupServer + mc *MasterClient +} + +func (s *resetDuringLookupServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + s.mc.resetVidMap() + return s.fakeLookupServer.LookupVolume(ctx, req) +} + +// An answer sent before a reset must not land in the new generation, which a +// gateway's FilerClient reads as the current master's view. +func TestLookupAnswerCrossingResetIsNotCached(t *testing.T) { + mc := NewMasterClient( + grpc.WithTransportCredentials(insecure.NewCredentials()), + "", "test", "", "", "", + pb.ServerDiscovery{}, + ) + srv := &resetDuringLookupServer{mc: mc} + mc.setCurrentMaster(startFakeMasterServer(t, srv)) + mc.grpcTimeout = 5 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + provider := &masterVolumeProvider{masterClient: mc} + result, err := provider.LookupVolumeIds(ctx, []string{"1"}) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if _, ok := result["1"]; !ok { + t.Fatal("expected volume 1 to be served") + } + if locations, found := mc.vidMap.currentLocations(1); found { + t.Fatalf("answer that crossed a reset was cached: %v", locations) + } +} diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go index 67fa4c83ae..3e1c78c786 100644 --- a/weed/wdclient/vid_map.go +++ b/weed/wdclient/vid_map.go @@ -225,6 +225,41 @@ func (vc *vidMap) hasLocation(vid uint32, url string) bool { return false } +// currentLocations returns what vid was learned at since the last reset, +// leaving out anything only an earlier generation reported. When both a +// regular and an ec entry are current, the regular copies serve, as in +// GetLocations. +func (vc *vidMap) currentLocations(vid uint32) ([]Location, bool) { + vc.RLock() + defer vc.RUnlock() + for _, vid2Locations := range []map[uint32]*locationsEntry{vc.vid2Locations, vc.ecVid2Locations} { + if entry, found := lookupEntry(vid2Locations, vid); found && entry.generation == vc.generation { + return entry.locations, true + } + } + return nil, false +} + +// setLocations makes locations vid's regular entry in the current +// generation, in place of whatever the entry held. +func (vc *vidMap) setLocations(vid uint32, locations []Location) { + vc.Lock() + defer vc.Unlock() + if entry, found := vc.vid2Locations[vid]; found { + vc.releaseEntry(entry) + } + if len(locations) == 0 { + delete(vc.vid2Locations, vid) + return + } + stored := make([]Location, len(locations)) + copy(stored, locations) + vc.vid2Locations[vid] = &locationsEntry{locations: stored, generation: vc.generation} + for _, loc := range stored { + vc.incrementServerRef(locationServerKey(loc)) + } +} + // lookupEntry returns vid's entry when it still holds locations. Callers must // hold the lock. func lookupEntry(vid2Locations map[uint32]*locationsEntry, vid uint32) (*locationsEntry, bool) { diff --git a/weed/wdclient/vidmap_client.go b/weed/wdclient/vidmap_client.go index 8e1b464021..d2a990af61 100644 --- a/weed/wdclient/vidmap_client.go +++ b/weed/wdclient/vidmap_client.go @@ -27,6 +27,10 @@ type VolumeLocationProvider interface { LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) } +// afterLookupRecheck, when set by a test, runs between a lookup's second cache +// check and its provider query. +var afterLookupRecheck func() + // vidMapClient provides volume location caching with pluggable lookup // It wraps the battle-tested vidMap with customizable volume lookup strategies type vidMapClient struct { @@ -210,6 +214,11 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI batchKey := strings.Join(needsLookup, ",") sfResult, err := vc.doSharedLookup(ctx, batchKey, func(ctx context.Context) (interface{}, error) { + // The epoch is taken before the cache is checked again: a confirmation + // or reset that lands after it either shows up in that check or stops + // the provider's answer from being cached. + fillEpoch := vc.mapEpoch() + // Double-check cache for volumes that might have been populated while waiting stillNeedLookup := make([]string, 0, len(needsLookup)) batchResult := make(map[string][]Location) @@ -223,6 +232,10 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI } } + if afterLookupRecheck != nil { + afterLookupRecheck() + } + if len(stillNeedLookup) == 0 { return batchResult, nil } @@ -230,7 +243,6 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI // Query provider with batched volume IDs glog.V(2).Infof("Looking up %d volumes from provider: %v", len(stillNeedLookup), stillNeedLookup) - fillEpoch := vc.mapEpoch() providerResults, err := vc.provider.LookupVolumeIds(ctx, stillNeedLookup) if err != nil { return batchResult, fmt.Errorf("provider lookup failed: %w", err) @@ -243,13 +255,7 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI continue } - var live []Location - for _, loc := range locations { - if vc.fillLocation(uint32(vid), loc, fillEpoch) { - live = append(live, loc) - } - } - + live := vc.fillLocations(uint32(vid), locations, fillEpoch) if len(live) > 0 { batchResult[vidString] = live } @@ -438,20 +444,36 @@ func (vc *vidMapClient) untombstone(vid uint32, loc Location) { } } -// fillLocation reports whether a provider answer may be served, and caches it -// unless a reset or clear happened since epoch. Holding recentDeletesMu across -// the check and the add keeps a concurrent removal from being undone. -func (vc *vidMapClient) fillLocation(vid uint32, loc Location, epoch uint64) bool { +// fillLocations returns the locations of a provider answer for vid that may +// be served, and caches them unless a reset, clear or confirmation happened +// since epoch. Holding recentDeletesMu across the check and the add keeps a +// concurrent removal or confirmation from being undone, and caches the +// answer for vid whole or not at all. +func (vc *vidMapClient) fillLocations(vid uint32, locations []Location, epoch uint64) (live []Location) { vc.recentDeletesMu.Lock() defer vc.recentDeletesMu.Unlock() vc.expireRecentDeletes(time.Now()) - if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found { - return false + for _, loc := range locations { + if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found { + continue + } + live = append(live, loc) + if vc.vidMapEpoch == epoch { + vc.vidMap.addLocation(vid, loc) + } } - if vc.vidMapEpoch == epoch { - vc.vidMap.addLocation(vid, loc) - } - return true + return live +} + +// confirmLocations caches what a master stream reports vid at now in place of +// what lookups cached for it, which may have come from a source that had not +// caught up with the master yet. Lookups already in flight answer from that +// same source, so the epoch moves on and they do not cache what they fetched. +func (vc *vidMapClient) confirmLocations(vid uint32, locations []Location) { + vc.recentDeletesMu.Lock() + defer vc.recentDeletesMu.Unlock() + vc.vidMapEpoch++ + vc.vidMap.setLocations(vid, locations) } // InvalidateCache removes all cached locations for a volume ID