diff --git a/weed/wdclient/filer_client.go b/weed/wdclient/filer_client.go index cf32328f0c..7f6f8df94f 100644 --- a/weed/wdclient/filer_client.go +++ b/weed/wdclient/filer_client.go @@ -528,8 +528,16 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve } // OnVolumeLocationChange applies master-pushed changes to the lookup-fed cache, -// so a retired server stops being tried before a read against it fails. Adds -// only lift the removal fence; locations are still learned by lookup. +// so a retired server stops being tried before a read against it fails. An +// add lifts the removal fence and caches the volume where the master client +// now knows it to be, replacing what a lookup cached for it. +// +// The replacement matters after a leader change. The reconnect clears this +// cache, and lookups refill it from the filer, whose own master client can +// still answer from the previous leader's view until the new leader has heard +// from the volume servers. A server lost with the old leader is then cached +// again, and the new leader, which never knew it, never pushes its removal. +// Its adds for the volumes that server held are what correct the refill. func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVids []uint32) { for _, vid := range deletedVids { fc.tombstone(vid, loc) @@ -537,6 +545,12 @@ func (fc *FilerClient) OnVolumeLocationChange(loc Location, addedVids, deletedVi } for _, vid := range addedVids { fc.untombstone(vid, loc) + if fc.masterClient == nil { + continue + } + if locations, found := fc.masterClient.vidMap.currentLocations(vid); found { + fc.confirmLocations(vid, locations) + } } } diff --git a/weed/wdclient/filer_client_test.go b/weed/wdclient/filer_client_test.go index 97e23dd9cb..a66d88efa2 100644 --- a/weed/wdclient/filer_client_test.go +++ b/weed/wdclient/filer_client_test.go @@ -561,3 +561,51 @@ func TestReconnectReplayLiftsFence(t *testing.T) { expectServed(t, fc, "5", a) } + +// A gateway that reconnects to a new leader drops its cache and refills it +// from a filer that may still answer with the old leader's view, a server +// that died with it included. The new leader never knew that server, so it +// never pushes its removal; its re-told adds are what replace the refill. +func TestLeaderChangeAddsReplaceStaleRefill(t *testing.T) { + dead := Location{Url: "dead:8080"} + live := Location{Url: "live:8080"} + other := Location{Url: "other:8080"} + provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}} + mc := moveClient() + fc := hookedFilerClient(mc, provider) + fc.masterClient = mc + pushVolumeLocation(mc, dead.Url, []uint32{5}, nil, nil, nil) + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + + mc.resetVidMap() + expectServed(t, fc, "5", dead, live) + + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + expectServed(t, fc, "5", live) + + pushVolumeLocation(mc, other.Url, []uint32{5}, nil, nil, nil) + expectServed(t, fc, "5", live, other) +} + +// A master add that lands between a lookup's second cache check and its query +// to a filer that has not caught up must not be undone by that filer's answer. +func TestConfirmDuringLookupIsNotUndone(t *testing.T) { + dead := Location{Url: "dead:8080"} + live := Location{Url: "live:8080"} + provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}} + mc := moveClient() + fc := hookedFilerClient(mc, provider) + fc.masterClient = mc + mc.resetVidMap() + + afterLookupRecheck = func() { + afterLookupRecheck = nil + pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil) + } + t.Cleanup(func() { afterLookupRecheck = nil }) + if _, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{"5"}); err != nil { + t.Fatalf("lookup: %v", err) + } + + expectServed(t, fc, "5", live) +} diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index e738c38871..6cc97f76fc 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -72,6 +72,7 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds [] func() error { result = make(map[string][]Location) lookupErrors = nil + epoch := p.masterClient.mapEpoch() // Per-attempt timeout bounds both master resolution and the RPC // so a single attempt cannot consume the entire retry budget. @@ -124,10 +125,9 @@ func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds [] ReadOnly: masterLoc.ReadOnly, ReadOnlyCanDelete: masterLoc.ReadOnlyCanDelete, } - // Update cache with the location - p.masterClient.addLocation(uint32(vid), loc) locations = append(locations, loc) } + p.masterClient.fillLocations(uint32(vid), locations, epoch) if len(locations) > 0 { result[vidOnly] = locations diff --git a/weed/wdclient/masterclient_retry_test.go b/weed/wdclient/masterclient_retry_test.go index 1b24b78c7c..8499858d53 100644 --- a/weed/wdclient/masterclient_retry_test.go +++ b/weed/wdclient/masterclient_retry_test.go @@ -117,3 +117,40 @@ func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) { t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed) } } + +type resetDuringLookupServer struct { + fakeLookupServer + mc *MasterClient +} + +func (s *resetDuringLookupServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + s.mc.resetVidMap() + return s.fakeLookupServer.LookupVolume(ctx, req) +} + +// An answer sent before a reset must not land in the new generation, which a +// gateway's FilerClient reads as the current master's view. +func TestLookupAnswerCrossingResetIsNotCached(t *testing.T) { + mc := NewMasterClient( + grpc.WithTransportCredentials(insecure.NewCredentials()), + "", "test", "", "", "", + pb.ServerDiscovery{}, + ) + srv := &resetDuringLookupServer{mc: mc} + mc.setCurrentMaster(startFakeMasterServer(t, srv)) + mc.grpcTimeout = 5 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + provider := &masterVolumeProvider{masterClient: mc} + result, err := provider.LookupVolumeIds(ctx, []string{"1"}) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if _, ok := result["1"]; !ok { + t.Fatal("expected volume 1 to be served") + } + if locations, found := mc.vidMap.currentLocations(1); found { + t.Fatalf("answer that crossed a reset was cached: %v", locations) + } +} diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go index 67fa4c83ae..3e1c78c786 100644 --- a/weed/wdclient/vid_map.go +++ b/weed/wdclient/vid_map.go @@ -225,6 +225,41 @@ func (vc *vidMap) hasLocation(vid uint32, url string) bool { return false } +// currentLocations returns what vid was learned at since the last reset, +// leaving out anything only an earlier generation reported. When both a +// regular and an ec entry are current, the regular copies serve, as in +// GetLocations. +func (vc *vidMap) currentLocations(vid uint32) ([]Location, bool) { + vc.RLock() + defer vc.RUnlock() + for _, vid2Locations := range []map[uint32]*locationsEntry{vc.vid2Locations, vc.ecVid2Locations} { + if entry, found := lookupEntry(vid2Locations, vid); found && entry.generation == vc.generation { + return entry.locations, true + } + } + return nil, false +} + +// setLocations makes locations vid's regular entry in the current +// generation, in place of whatever the entry held. +func (vc *vidMap) setLocations(vid uint32, locations []Location) { + vc.Lock() + defer vc.Unlock() + if entry, found := vc.vid2Locations[vid]; found { + vc.releaseEntry(entry) + } + if len(locations) == 0 { + delete(vc.vid2Locations, vid) + return + } + stored := make([]Location, len(locations)) + copy(stored, locations) + vc.vid2Locations[vid] = &locationsEntry{locations: stored, generation: vc.generation} + for _, loc := range stored { + vc.incrementServerRef(locationServerKey(loc)) + } +} + // lookupEntry returns vid's entry when it still holds locations. Callers must // hold the lock. func lookupEntry(vid2Locations map[uint32]*locationsEntry, vid uint32) (*locationsEntry, bool) { diff --git a/weed/wdclient/vidmap_client.go b/weed/wdclient/vidmap_client.go index 8e1b464021..d2a990af61 100644 --- a/weed/wdclient/vidmap_client.go +++ b/weed/wdclient/vidmap_client.go @@ -27,6 +27,10 @@ type VolumeLocationProvider interface { LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) } +// afterLookupRecheck, when set by a test, runs between a lookup's second cache +// check and its provider query. +var afterLookupRecheck func() + // vidMapClient provides volume location caching with pluggable lookup // It wraps the battle-tested vidMap with customizable volume lookup strategies type vidMapClient struct { @@ -210,6 +214,11 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI batchKey := strings.Join(needsLookup, ",") sfResult, err := vc.doSharedLookup(ctx, batchKey, func(ctx context.Context) (interface{}, error) { + // The epoch is taken before the cache is checked again: a confirmation + // or reset that lands after it either shows up in that check or stops + // the provider's answer from being cached. + fillEpoch := vc.mapEpoch() + // Double-check cache for volumes that might have been populated while waiting stillNeedLookup := make([]string, 0, len(needsLookup)) batchResult := make(map[string][]Location) @@ -223,6 +232,10 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI } } + if afterLookupRecheck != nil { + afterLookupRecheck() + } + if len(stillNeedLookup) == 0 { return batchResult, nil } @@ -230,7 +243,6 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI // Query provider with batched volume IDs glog.V(2).Infof("Looking up %d volumes from provider: %v", len(stillNeedLookup), stillNeedLookup) - fillEpoch := vc.mapEpoch() providerResults, err := vc.provider.LookupVolumeIds(ctx, stillNeedLookup) if err != nil { return batchResult, fmt.Errorf("provider lookup failed: %w", err) @@ -243,13 +255,7 @@ func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeI continue } - var live []Location - for _, loc := range locations { - if vc.fillLocation(uint32(vid), loc, fillEpoch) { - live = append(live, loc) - } - } - + live := vc.fillLocations(uint32(vid), locations, fillEpoch) if len(live) > 0 { batchResult[vidString] = live } @@ -438,20 +444,36 @@ func (vc *vidMapClient) untombstone(vid uint32, loc Location) { } } -// fillLocation reports whether a provider answer may be served, and caches it -// unless a reset or clear happened since epoch. Holding recentDeletesMu across -// the check and the add keeps a concurrent removal from being undone. -func (vc *vidMapClient) fillLocation(vid uint32, loc Location, epoch uint64) bool { +// fillLocations returns the locations of a provider answer for vid that may +// be served, and caches them unless a reset, clear or confirmation happened +// since epoch. Holding recentDeletesMu across the check and the add keeps a +// concurrent removal or confirmation from being undone, and caches the +// answer for vid whole or not at all. +func (vc *vidMapClient) fillLocations(vid uint32, locations []Location, epoch uint64) (live []Location) { vc.recentDeletesMu.Lock() defer vc.recentDeletesMu.Unlock() vc.expireRecentDeletes(time.Now()) - if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found { - return false + for _, loc := range locations { + if _, found := vc.recentDeletes[vidLocationKey{vid: vid, url: loc.Url}]; found { + continue + } + live = append(live, loc) + if vc.vidMapEpoch == epoch { + vc.vidMap.addLocation(vid, loc) + } } - if vc.vidMapEpoch == epoch { - vc.vidMap.addLocation(vid, loc) - } - return true + return live +} + +// confirmLocations caches what a master stream reports vid at now in place of +// what lookups cached for it, which may have come from a source that had not +// caught up with the master yet. Lookups already in flight answer from that +// same source, so the epoch moves on and they do not cache what they fetched. +func (vc *vidMapClient) confirmLocations(vid uint32, locations []Location) { + vc.recentDeletesMu.Lock() + defer vc.recentDeletesMu.Unlock() + vc.vidMapEpoch++ + vc.vidMap.setLocations(vid, locations) } // InvalidateCache removes all cached locations for a volume ID