mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 17:55:26 +01:00
* wdclient: let master-pushed adds replace the gateway's lookup-fed locations After a master leader change the S3 gateway clears its FilerClient cache and refills it through filer lookups. The filer's own master client may still answer from the previous leader's generation until the new leader has heard from the volume servers, so a volume server lost together with the old leader is cached again. The new leader never knew that server and never pushes its removal, and the gateway keeps trying it until reads fail on it volume by volume, one 30 s dial timeout at a time. The gateway's master client relearns every volume from the new leader's adds. On each add, cache the volume where the master client now knows it to be, in place of what a lookup cached, and keep lookups already in flight from caching their answer over it. A provider answer for a volume is now cached whole or not at all. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: take a lookup's fill epoch before its second cache check A master add that landed between the second cache check and the read of the fill epoch was invisible to both: the check had already missed, and the epoch the lookup then read already included the confirmation, so a filer answer that still named the lost server was merged back into the confirmed entry. Taking the epoch first means such an add either shows up in the check or stops the answer from being cached. The same order covers a reset or clear in that gap. TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook at that point and fails with the old order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: gate the master lookup provider's cache writes on the map epoch The provider wrote each answer straight into the vid map, so a lookup sent before a master reset and answered after it landed in the new generation, which a gateway's FilerClient now reads as the current master's view. The answer now goes through fillLocations, which drops it once a reset passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: keep the vidMapClient doc comment on its type Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
612 lines
21 KiB
Go
612 lines
21 KiB
Go
package wdclient
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
)
|
|
|
|
func newTestFilerClient(addrs ...pb.ServerAddress) *FilerClient {
|
|
health := make([]*filerHealth, len(addrs))
|
|
for i := range health {
|
|
health[i] = &filerHealth{}
|
|
}
|
|
return &FilerClient{
|
|
filerAddresses: addrs,
|
|
filerHealth: health,
|
|
vidMapClient: newVidMapClient(nil, "", 0),
|
|
}
|
|
}
|
|
|
|
func filerAddressList(fc *FilerClient) []pb.ServerAddress {
|
|
out := make([]pb.ServerAddress, len(fc.filerAddresses))
|
|
copy(out, fc.filerAddresses)
|
|
return out
|
|
}
|
|
|
|
func TestApplyDiscoveredFilersPrunesStaleAddress(t *testing.T) {
|
|
a := pb.ServerAddress("10.0.0.1:18888")
|
|
b := pb.ServerAddress("10.0.0.2:18888") // gets replaced by c
|
|
c := pb.ServerAddress("10.0.0.3:18888")
|
|
|
|
fc := newTestFilerClient(a, b)
|
|
// Give b a non-trivial failure count so we can confirm it leaves with its health.
|
|
atomic.StoreInt32(&fc.filerHealth[1].failureCount, 7)
|
|
// Give a a known failure count so we can confirm survivor health is preserved.
|
|
atomic.StoreInt32(&fc.filerHealth[0].failureCount, 2)
|
|
atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b
|
|
|
|
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
|
|
a: {},
|
|
c: {},
|
|
})
|
|
|
|
got := filerAddressList(fc)
|
|
if len(got) != 2 || got[0] != a || got[1] != c {
|
|
t.Fatalf("expected [%s %s], got %v", a, c, got)
|
|
}
|
|
if len(fc.filerHealth) != 2 {
|
|
t.Fatalf("expected 2 health entries, got %d", len(fc.filerHealth))
|
|
}
|
|
if got := atomic.LoadInt32(&fc.filerHealth[0].failureCount); got != 2 {
|
|
t.Errorf("survivor health was reset: want 2, got %d", got)
|
|
}
|
|
if got := atomic.LoadInt32(&fc.filerHealth[1].failureCount); got != 0 {
|
|
t.Errorf("newly added filer should start with fresh health, got failureCount=%d", got)
|
|
}
|
|
// Active filer (b) disappeared; index must reset rather than dangle.
|
|
if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
|
|
t.Errorf("expected filerIndex reset to 0 after active filer removed, got %d", idx)
|
|
}
|
|
}
|
|
|
|
func TestApplyDiscoveredFilersKeepsIndexOnSurvivor(t *testing.T) {
|
|
a := pb.ServerAddress("10.0.0.1:18888") // gets replaced
|
|
b := pb.ServerAddress("10.0.0.2:18888") // active, survives
|
|
c := pb.ServerAddress("10.0.0.3:18888") // new
|
|
|
|
fc := newTestFilerClient(a, b)
|
|
atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b
|
|
|
|
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
|
|
b: {},
|
|
c: {},
|
|
})
|
|
|
|
got := filerAddressList(fc)
|
|
if len(got) != 2 || got[0] != b || got[1] != c {
|
|
t.Fatalf("expected [%s %s], got %v", b, c, got)
|
|
}
|
|
// b moved to index 0 after a was pruned; index should follow.
|
|
if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
|
|
t.Errorf("expected filerIndex to follow surviving active filer to position 0, got %d", idx)
|
|
}
|
|
}
|
|
|
|
func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) {
|
|
a := pb.ServerAddress("10.0.0.1:18888")
|
|
b := pb.ServerAddress("10.0.0.2:18888")
|
|
|
|
fc := newTestFilerClient(a, b)
|
|
originalHealthA := fc.filerHealth[0]
|
|
originalHealthB := fc.filerHealth[1]
|
|
|
|
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
|
|
a: {},
|
|
b: {},
|
|
})
|
|
|
|
if fc.filerHealth[0] != originalHealthA || fc.filerHealth[1] != originalHealthB {
|
|
t.Errorf("no-op refresh should not reallocate health entries")
|
|
}
|
|
}
|
|
|
|
func filerUpdate(addr pb.ServerAddress, isAdd bool) *master_pb.ClusterNodeUpdate {
|
|
return &master_pb.ClusterNodeUpdate{NodeType: cluster.FilerType, Address: string(addr), IsAdd: isAdd}
|
|
}
|
|
|
|
// A rolling restart replaces every filer within one discovery interval; the
|
|
// pushed updates must keep the list live without waiting for the next poll.
|
|
func TestOnPeerUpdateTracksRollingReplacement(t *testing.T) {
|
|
old1 := pb.ServerAddress("10.0.0.1:8888")
|
|
old2 := pb.ServerAddress("10.0.0.2:8888")
|
|
new1 := pb.ServerAddress("10.0.1.1:8888")
|
|
new2 := pb.ServerAddress("10.0.1.2:8888")
|
|
|
|
fc := newTestFilerClient(old1, old2)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(old1, false), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(new1, true), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(old2, false), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(new2, true), time.Now())
|
|
|
|
got := filerAddressList(fc)
|
|
if len(got) != 2 || got[0] != new1 || got[1] != new2 {
|
|
t.Fatalf("expected [%s %s], got %v", new1, new2, got)
|
|
}
|
|
}
|
|
|
|
func TestOnPeerUpdateKeepsLastFiler(t *testing.T) {
|
|
only := pb.ServerAddress("10.0.0.1:8888")
|
|
fc := newTestFilerClient(only)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(only, false), time.Now())
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != only {
|
|
t.Fatalf("removing the last filer must keep it, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestOnPeerUpdateIgnoresOtherNodeTypes(t *testing.T) {
|
|
a := pb.ServerAddress("10.0.0.1:8888")
|
|
fc := newTestFilerClient(a)
|
|
|
|
fc.OnPeerUpdate(&master_pb.ClusterNodeUpdate{NodeType: cluster.S3Type, Address: "10.0.0.9:18333", IsAdd: true}, time.Now())
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != a {
|
|
t.Fatalf("non-filer update changed the list: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestOnPeerUpdateDuplicateAddKeepsHealth(t *testing.T) {
|
|
a := pb.ServerAddress("10.0.0.1:8888")
|
|
fc := newTestFilerClient(a)
|
|
atomic.StoreInt32(&fc.filerHealth[0].failureCount, 2)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(a, true), time.Now())
|
|
|
|
if len(fc.filerHealth) != 1 || atomic.LoadInt32(&fc.filerHealth[0].failureCount) != 2 {
|
|
t.Fatalf("re-announced filer lost its health state")
|
|
}
|
|
}
|
|
|
|
func TestDiscoverySnapshotTakenBeforePushIsDiscarded(t *testing.T) {
|
|
old := pb.ServerAddress("10.0.0.1:8888")
|
|
joined := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(old)
|
|
|
|
generation := fc.peerUpdateGeneration()
|
|
fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
|
|
fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{old: {}}, generation)
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != joined {
|
|
t.Fatalf("stale snapshot overwrote pushed membership: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverySnapshotWithoutInterveningPushIsApplied(t *testing.T) {
|
|
old := pb.ServerAddress("10.0.0.1:8888")
|
|
replacement := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(old)
|
|
|
|
fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, fc.peerUpdateGeneration())
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != replacement {
|
|
t.Fatalf("expected snapshot to replace list, got %v", got)
|
|
}
|
|
}
|
|
|
|
// A leave suppressed to keep the last filer is deferred, then honored as soon
|
|
// as a replacement joins, so the departed address stops being a candidate.
|
|
func TestOnPeerUpdateDeferredLeaveFlushesOnJoin(t *testing.T) {
|
|
old := pb.ServerAddress("10.0.0.1:8888")
|
|
joined := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(old)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != old {
|
|
t.Fatalf("leave of the last filer must be deferred, got %v", got)
|
|
}
|
|
if len(fc.deferredLeaves) != 1 {
|
|
t.Fatalf("expected the leave to be deferred, got %v", fc.deferredLeaves)
|
|
}
|
|
|
|
fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != joined {
|
|
t.Fatalf("deferred leave should flush on join, got %v", got)
|
|
}
|
|
if len(fc.deferredLeaves) != 0 {
|
|
t.Fatalf("deferred leaves should be empty after flush, got %v", fc.deferredLeaves)
|
|
}
|
|
}
|
|
|
|
// A pushed add for an already-known filer changes nothing and must not bump
|
|
// the generation that guards an in-flight discovery snapshot.
|
|
func TestOnPeerUpdateNoopDoesNotDiscardSnapshot(t *testing.T) {
|
|
old := pb.ServerAddress("10.0.0.1:8888")
|
|
replacement := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(old)
|
|
|
|
generation := fc.peerUpdateGeneration()
|
|
fc.OnPeerUpdate(filerUpdate(old, true), time.Now())
|
|
fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, generation)
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != replacement {
|
|
t.Fatalf("no-op push should not discard the snapshot, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestOnPeerUpdateRejoinCancelsDeferredLeave(t *testing.T) {
|
|
restarted := pb.ServerAddress("10.0.0.1:8888")
|
|
joined := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(restarted)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(restarted, false), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(restarted, true), time.Now())
|
|
fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())
|
|
|
|
if got := filerAddressList(fc); len(got) != 2 || got[0] != restarted || got[1] != joined {
|
|
t.Fatalf("rejoined filer was dropped by its stale deferred leave: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverySnapshotClearsDeferredLeaves(t *testing.T) {
|
|
departed := pb.ServerAddress("10.0.0.1:8888")
|
|
replacement := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(departed)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(departed, false), time.Now())
|
|
fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, fc.peerUpdateGeneration())
|
|
fc.OnPeerUpdate(filerUpdate(departed, true), time.Now())
|
|
|
|
if got := filerAddressList(fc); len(got) != 2 || got[0] != replacement || got[1] != departed {
|
|
t.Fatalf("rejoined filer was dropped by a deferred leave the poll should have cleared: %v", got)
|
|
}
|
|
}
|
|
|
|
// A poll that started after a deferred leave but before the filer rejoined can
|
|
// return a snapshot lacking the rejoined filer; the rejoin must bump the
|
|
// generation so that snapshot is discarded.
|
|
func TestOnPeerUpdateRejoinBumpsGeneration(t *testing.T) {
|
|
old := pb.ServerAddress("10.0.0.1:8888")
|
|
other := pb.ServerAddress("10.0.1.1:8888")
|
|
fc := newTestFilerClient(old)
|
|
|
|
fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
|
|
generation := fc.peerUpdateGeneration()
|
|
fc.OnPeerUpdate(filerUpdate(old, true), time.Now())
|
|
fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{other: {}}, generation)
|
|
|
|
if got := filerAddressList(fc); len(got) != 1 || got[0] != old {
|
|
t.Fatalf("rejoined filer was pruned by an in-flight snapshot: %v", got)
|
|
}
|
|
}
|
|
|
|
// stallingFiler answers LookupVolume only when the call is abandoned.
|
|
type stallingFiler struct {
|
|
filer_pb.UnimplementedSeaweedFilerServer
|
|
}
|
|
|
|
func (s *stallingFiler) LookupVolume(ctx context.Context, _ *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
func startStallingFiler(t *testing.T) pb.ServerAddress {
|
|
t.Helper()
|
|
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
grpcServer := grpc.NewServer()
|
|
filer_pb.RegisterSeaweedFilerServer(grpcServer, &stallingFiler{})
|
|
go func() { _ = grpcServer.Serve(lis) }()
|
|
t.Cleanup(grpcServer.Stop)
|
|
|
|
_, port, _ := net.SplitHostPort(lis.Addr().String())
|
|
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
|
|
}
|
|
|
|
// lapsedDeadlineCtx mimics the moment after a caller's deadline has passed but
|
|
// before the runtime marks the context done.
|
|
type lapsedDeadlineCtx struct{ context.Context }
|
|
|
|
func (lapsedDeadlineCtx) Deadline() (time.Time, bool) { return time.Now().Add(-time.Second), true }
|
|
func (lapsedDeadlineCtx) Done() <-chan struct{} { return nil }
|
|
func (lapsedDeadlineCtx) Err() error { return nil }
|
|
|
|
// A caller giving up must not count as a filer failure, the filer's own timeout must.
|
|
func TestLookupVolumeIdsCountsOnlyFilerFailures(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
callerCtx func() (context.Context, context.CancelFunc)
|
|
grpcTimeout time.Duration
|
|
wantErr error
|
|
wantFailures int32
|
|
}{
|
|
{
|
|
name: "caller cancelled",
|
|
callerCtx: func() (context.Context, context.CancelFunc) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
time.AfterFunc(50*time.Millisecond, cancel)
|
|
return ctx, cancel
|
|
},
|
|
grpcTimeout: 5 * time.Second,
|
|
wantErr: context.Canceled,
|
|
wantFailures: 0,
|
|
},
|
|
{
|
|
name: "caller deadline",
|
|
callerCtx: func() (context.Context, context.CancelFunc) {
|
|
return context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
},
|
|
grpcTimeout: 5 * time.Second,
|
|
wantErr: context.DeadlineExceeded,
|
|
wantFailures: 0,
|
|
},
|
|
{
|
|
name: "caller deadline already elapsed",
|
|
callerCtx: func() (context.Context, context.CancelFunc) {
|
|
return lapsedDeadlineCtx{context.Background()}, func() {}
|
|
},
|
|
grpcTimeout: 5 * time.Second,
|
|
wantErr: context.DeadlineExceeded,
|
|
wantFailures: 0,
|
|
},
|
|
{
|
|
name: "filer timeout",
|
|
callerCtx: func() (context.Context, context.CancelFunc) {
|
|
return context.WithCancel(context.Background())
|
|
},
|
|
grpcTimeout: 50 * time.Millisecond,
|
|
wantFailures: 1,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
fc := newTestFilerClient(startStallingFiler(t))
|
|
fc.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
|
|
fc.grpcTimeout = tt.grpcTimeout
|
|
fc.maxRetries = 1
|
|
fc.failureThreshold = 3
|
|
fc.resetTimeout = time.Minute
|
|
|
|
ctx, cancel := tt.callerCtx()
|
|
defer cancel()
|
|
_, err := (&filerVolumeProvider{filerClient: fc}).LookupVolumeIds(ctx, []string{"4"})
|
|
|
|
if err == nil {
|
|
t.Fatal("lookup against a stalling filer succeeded")
|
|
}
|
|
if tt.wantErr != nil && !errors.Is(err, tt.wantErr) {
|
|
t.Errorf("err = %v, want %v", err, tt.wantErr)
|
|
}
|
|
if got := atomic.LoadInt32(&fc.filerHealth[0].failureCount); got != tt.wantFailures {
|
|
t.Errorf("failureCount = %d, want %d", got, tt.wantFailures)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOnVolumeLocationChangeDropsRetiredServer(t *testing.T) {
|
|
fc := newTestFilerClient()
|
|
retired := Location{Url: "retired:8080"}
|
|
live := Location{Url: "live:8080"}
|
|
fc.addLocation(7, retired)
|
|
fc.addLocation(7, live)
|
|
fc.addLocation(8, retired)
|
|
|
|
fc.OnVolumeLocationChange(retired, nil, []uint32{7, 8, 9})
|
|
|
|
locations, found := fc.GetLocations(7)
|
|
if !found || len(locations) != 1 || locations[0].Url != live.Url {
|
|
t.Errorf("vid 7 kept only the surviving location: found=%v %v", found, locations)
|
|
}
|
|
if locations, found := fc.GetLocations(8); found && len(locations) > 0 {
|
|
t.Errorf("vid 8 lost its last location but stayed cached: %v", locations)
|
|
}
|
|
}
|
|
|
|
func TestOnVidMapResetClearsCachedLocations(t *testing.T) {
|
|
fc := newTestFilerClient()
|
|
fc.addLocation(7, Location{Url: "live:8080"})
|
|
|
|
fc.OnVidMapReset()
|
|
|
|
if locations, found := fc.GetLocations(7); found && len(locations) > 0 {
|
|
t.Errorf("vid 7 survived the reset: %v", locations)
|
|
}
|
|
}
|
|
|
|
func TestOnVolumeLocationChangeReaddClearsTombstone(t *testing.T) {
|
|
retired := Location{Url: "retired:8080"}
|
|
fc := newTestFilerClient()
|
|
fc.vidMapClient = newVidMapClient(&testLocationProvider{
|
|
locations: map[string][]Location{"5": {retired}},
|
|
}, "", 0)
|
|
ctx := context.Background()
|
|
|
|
fc.OnVolumeLocationChange(retired, nil, []uint32{5})
|
|
if locations, _ := fc.LookupVolumeIdsWithFallback(ctx, []string{"5"}); len(locations["5"]) > 0 {
|
|
t.Fatalf("removed location still served: %v", locations)
|
|
}
|
|
|
|
fc.OnVolumeLocationChange(retired, []uint32{5}, nil)
|
|
locations, err := fc.LookupVolumeIdsWithFallback(ctx, []string{"5"})
|
|
if err != nil || len(locations["5"]) != 1 || locations["5"][0].Url != retired.Url {
|
|
t.Fatalf("re-added location not served again: %v, err %v", locations, err)
|
|
}
|
|
}
|
|
|
|
func hookedFilerClient(mc *MasterClient, provider VolumeLocationProvider) *FilerClient {
|
|
fc := newTestFilerClient()
|
|
fc.vidMapClient = newVidMapClient(provider, "", 0)
|
|
mc.SetOnVolumeLocationChangeFn(fc.OnVolumeLocationChange)
|
|
mc.SetOnVidMapResetFn(fc.OnVidMapReset)
|
|
return fc
|
|
}
|
|
|
|
func expectServed(t *testing.T, fc *FilerClient, vid string, want ...Location) {
|
|
t.Helper()
|
|
locations, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{vid})
|
|
got := locations[vid]
|
|
if len(want) == 0 {
|
|
if len(got) != 0 {
|
|
t.Fatalf("volume %s still served from %v", vid, got)
|
|
}
|
|
return
|
|
}
|
|
if err != nil || len(got) != len(want) {
|
|
t.Fatalf("volume %s served from %v, err %v, want %v", vid, got, err, want)
|
|
}
|
|
for i := range want {
|
|
if got[i].Url != want[i].Url {
|
|
t.Fatalf("volume %s served from %v, want %v", vid, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ec.decode mounts the regular copy on a shard server before the shards come
|
|
// off it, so the later shard removal must leave that copy servable.
|
|
func TestDecodedCopyOutlivesShardRemoval(t *testing.T) {
|
|
a := Location{Url: "a:8080"}
|
|
b := Location{Url: "b:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {a, b}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
|
|
pushVolumeLocation(mc, b.Url, nil, nil, []uint32{5}, nil)
|
|
expectServed(t, fc, "5", a, b)
|
|
|
|
pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
|
|
pushVolumeLocation(mc, b.Url, nil, nil, nil, []uint32{5})
|
|
pushVolumeLocation(mc, a.Url, nil, nil, nil, []uint32{5})
|
|
|
|
provider.locations["5"] = []Location{a, b}
|
|
expectServed(t, fc, "5", a)
|
|
}
|
|
|
|
// ec.encode registers the shards before the regular copy is deleted from the
|
|
// same server, so the server stays a read candidate for the ec volume.
|
|
func TestShardServerOutlivesRegularRemoval(t *testing.T) {
|
|
a := Location{Url: "a:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {a}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
|
|
expectServed(t, fc, "5", a)
|
|
|
|
pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
|
|
pushVolumeLocation(mc, a.Url, nil, []uint32{5}, nil, nil)
|
|
|
|
expectServed(t, fc, "5", a)
|
|
}
|
|
|
|
// The filer caches shard servers like regular copies, so the last shard
|
|
// leaving a server retires it from the cache too.
|
|
func TestRetiredShardServerLeavesCache(t *testing.T) {
|
|
a := Location{Url: "a:8080"}
|
|
b := Location{Url: "b:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {a, b}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
|
|
pushVolumeLocation(mc, b.Url, nil, nil, []uint32{5}, nil)
|
|
expectServed(t, fc, "5", a, b)
|
|
|
|
pushVolumeLocation(mc, a.Url, nil, nil, nil, []uint32{5})
|
|
|
|
if cached, _ := fc.GetLocations(5); len(cached) != 1 || cached[0].Url != b.Url {
|
|
t.Fatalf("cache holds %v, want only %v", cached, b)
|
|
}
|
|
fc.InvalidateCache("5,01")
|
|
expectServed(t, fc, "5", b)
|
|
}
|
|
|
|
// A volume moved to another server while the old one stays up: the old
|
|
// server would proxy the read and succeed, so no failed read would ever
|
|
// evict it. The pushed removal has to.
|
|
func TestMovedVolumeLeavesOldServer(t *testing.T) {
|
|
old := Location{Url: "old:8080"}
|
|
moved := Location{Url: "new:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {old}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
pushVolumeLocation(mc, old.Url, []uint32{5}, nil, nil, nil)
|
|
expectServed(t, fc, "5", old)
|
|
|
|
pushVolumeLocation(mc, moved.Url, []uint32{5}, nil, nil, nil)
|
|
pushVolumeLocation(mc, old.Url, nil, []uint32{5}, nil, nil)
|
|
|
|
provider.locations["5"] = []Location{old, moved}
|
|
expectServed(t, fc, "5", moved)
|
|
}
|
|
|
|
// A server back at the same URL after a master reconnect is re-told as an
|
|
// add, which lifts the fence its earlier removal left.
|
|
func TestReconnectReplayLiftsFence(t *testing.T) {
|
|
a := Location{Url: "a:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {a}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
|
|
pushVolumeLocation(mc, a.Url, nil, []uint32{5}, nil, nil)
|
|
expectServed(t, fc, "5")
|
|
|
|
mc.resetVidMap()
|
|
pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
|
|
|
|
expectServed(t, fc, "5", a)
|
|
}
|
|
|
|
// A gateway that reconnects to a new leader drops its cache and refills it
|
|
// from a filer that may still answer with the old leader's view, a server
|
|
// that died with it included. The new leader never knew that server, so it
|
|
// never pushes its removal; its re-told adds are what replace the refill.
|
|
func TestLeaderChangeAddsReplaceStaleRefill(t *testing.T) {
|
|
dead := Location{Url: "dead:8080"}
|
|
live := Location{Url: "live:8080"}
|
|
other := Location{Url: "other:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
fc.masterClient = mc
|
|
pushVolumeLocation(mc, dead.Url, []uint32{5}, nil, nil, nil)
|
|
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
|
|
|
|
mc.resetVidMap()
|
|
expectServed(t, fc, "5", dead, live)
|
|
|
|
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
|
|
expectServed(t, fc, "5", live)
|
|
|
|
pushVolumeLocation(mc, other.Url, []uint32{5}, nil, nil, nil)
|
|
expectServed(t, fc, "5", live, other)
|
|
}
|
|
|
|
// A master add that lands between a lookup's second cache check and its query
|
|
// to a filer that has not caught up must not be undone by that filer's answer.
|
|
func TestConfirmDuringLookupIsNotUndone(t *testing.T) {
|
|
dead := Location{Url: "dead:8080"}
|
|
live := Location{Url: "live:8080"}
|
|
provider := &testLocationProvider{locations: map[string][]Location{"5": {dead, live}}}
|
|
mc := moveClient()
|
|
fc := hookedFilerClient(mc, provider)
|
|
fc.masterClient = mc
|
|
mc.resetVidMap()
|
|
|
|
afterLookupRecheck = func() {
|
|
afterLookupRecheck = nil
|
|
pushVolumeLocation(mc, live.Url, []uint32{5}, nil, nil, nil)
|
|
}
|
|
t.Cleanup(func() { afterLookupRecheck = nil })
|
|
if _, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{"5"}); err != nil {
|
|
t.Fatalf("lookup: %v", err)
|
|
}
|
|
|
|
expectServed(t, fc, "5", live)
|
|
}
|