seaweedfs/weed/wdclient/filer_client_test.go

package wdclient

import (
	"context"
	"errors"
	"fmt"
	"net"
	"sync/atomic"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

func newTestFilerClient(addrs ...pb.ServerAddress) *FilerClient {
	health := make([]*filerHealth, len(addrs))
	for i := range health {
		health[i] = &filerHealth{}
	}
	return &FilerClient{
		filerAddresses: addrs,
		filerHealth:    health,
		vidMapClient:   newVidMapClient(nil, "", 0),
	}
}

func filerAddressList(fc *FilerClient) []pb.ServerAddress {
	out := make([]pb.ServerAddress, len(fc.filerAddresses))
	copy(out, fc.filerAddresses)
	return out
}

func TestApplyDiscoveredFilersPrunesStaleAddress(t *testing.T) {
	a := pb.ServerAddress("10.0.0.1:18888")
	b := pb.ServerAddress("10.0.0.2:18888") // gets replaced by c
	c := pb.ServerAddress("10.0.0.3:18888")

	fc := newTestFilerClient(a, b)
	// Give b a non-trivial failure count so we can confirm it leaves with its health.
	atomic.StoreInt32(&fc.filerHealth[1].failureCount, 7)
	// Give a a known failure count so we can confirm survivor health is preserved.
	atomic.StoreInt32(&fc.filerHealth[0].failureCount, 2)
	atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b

	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
		a: {},
		c: {},
	})

	got := filerAddressList(fc)
	if len(got) != 2 || got[0] != a || got[1] != c {
		t.Fatalf("expected [%s %s], got %v", a, c, got)
	}
	if len(fc.filerHealth) != 2 {
		t.Fatalf("expected 2 health entries, got %d", len(fc.filerHealth))
	}
	if got := atomic.LoadInt32(&fc.filerHealth[0].failureCount); got != 2 {
		t.Errorf("survivor health was reset: want 2, got %d", got)
	}
	if got := atomic.LoadInt32(&fc.filerHealth[1].failureCount); got != 0 {
		t.Errorf("newly added filer should start with fresh health, got failureCount=%d", got)
	}
	// Active filer (b) disappeared; index must reset rather than dangle.
	if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
		t.Errorf("expected filerIndex reset to 0 after active filer removed, got %d", idx)
	}
}

func TestApplyDiscoveredFilersKeepsIndexOnSurvivor(t *testing.T) {
	a := pb.ServerAddress("10.0.0.1:18888") // gets replaced
	b := pb.ServerAddress("10.0.0.2:18888") // active, survives
	c := pb.ServerAddress("10.0.0.3:18888") // new

	fc := newTestFilerClient(a, b)
	atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b

	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
		b: {},
		c: {},
	})

	got := filerAddressList(fc)
	if len(got) != 2 || got[0] != b || got[1] != c {
		t.Fatalf("expected [%s %s], got %v", b, c, got)
	}
	// b moved to index 0 after a was pruned; index should follow.
	if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
		t.Errorf("expected filerIndex to follow surviving active filer to position 0, got %d", idx)
	}
}

func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) {
	a := pb.ServerAddress("10.0.0.1:18888")
	b := pb.ServerAddress("10.0.0.2:18888")

	fc := newTestFilerClient(a, b)
	originalHealthA := fc.filerHealth[0]
	originalHealthB := fc.filerHealth[1]

	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
		a: {},
		b: {},
	})

	if fc.filerHealth[0] != originalHealthA || fc.filerHealth[1] != originalHealthB {
		t.Errorf("no-op refresh should not reallocate health entries")
	}
}

func filerUpdate(addr pb.ServerAddress, isAdd bool) *master_pb.ClusterNodeUpdate {
	return &master_pb.ClusterNodeUpdate{NodeType: cluster.FilerType, Address: string(addr), IsAdd: isAdd}
}

// A rolling restart replaces every filer within one discovery interval; the
// pushed updates must keep the list live without waiting for the next poll.
func TestOnPeerUpdateTracksRollingReplacement(t *testing.T) {
	old1 := pb.ServerAddress("10.0.0.1:8888")
	old2 := pb.ServerAddress("10.0.0.2:8888")
	new1 := pb.ServerAddress("10.0.1.1:8888")
	new2 := pb.ServerAddress("10.0.1.2:8888")

	fc := newTestFilerClient(old1, old2)

	fc.OnPeerUpdate(filerUpdate(old1, false), time.Now())
	fc.OnPeerUpdate(filerUpdate(new1, true), time.Now())
	fc.OnPeerUpdate(filerUpdate(old2, false), time.Now())
	fc.OnPeerUpdate(filerUpdate(new2, true), time.Now())

	got := filerAddressList(fc)
	if len(got) != 2 || got[0] != new1 || got[1] != new2 {
		t.Fatalf("expected [%s %s], got %v", new1, new2, got)
	}
}

func TestOnPeerUpdateKeepsLastFiler(t *testing.T) {
	only := pb.ServerAddress("10.0.0.1:8888")
	fc := newTestFilerClient(only)

	fc.OnPeerUpdate(filerUpdate(only, false), time.Now())

	if got := filerAddressList(fc); len(got) != 1 || got[0] != only {
		t.Fatalf("removing the last filer must keep it, got %v", got)
	}
}

func TestOnPeerUpdateIgnoresOtherNodeTypes(t *testing.T) {
	a := pb.ServerAddress("10.0.0.1:8888")
	fc := newTestFilerClient(a)

	fc.OnPeerUpdate(&master_pb.ClusterNodeUpdate{NodeType: cluster.S3Type, Address: "10.0.0.9:18333", IsAdd: true}, time.Now())

	if got := filerAddressList(fc); len(got) != 1 || got[0] != a {
		t.Fatalf("non-filer update changed the list: %v", got)
	}
}

func TestOnPeerUpdateDuplicateAddKeepsHealth(t *testing.T) {
	a := pb.ServerAddress("10.0.0.1:8888")
	fc := newTestFilerClient(a)
	atomic.StoreInt32(&fc.filerHealth[0].failureCount, 2)

	fc.OnPeerUpdate(filerUpdate(a, true), time.Now())

	if len(fc.filerHealth) != 1 || atomic.LoadInt32(&fc.filerHealth[0].failureCount) != 2 {
		t.Fatalf("re-announced filer lost its health state")
	}
}

func TestDiscoverySnapshotTakenBeforePushIsDiscarded(t *testing.T) {
	old := pb.ServerAddress("10.0.0.1:8888")
	joined := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(old)

	generation := fc.peerUpdateGeneration()
	fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())
	fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
	fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{old: {}}, generation)

	if got := filerAddressList(fc); len(got) != 1 || got[0] != joined {
		t.Fatalf("stale snapshot overwrote pushed membership: %v", got)
	}
}

func TestDiscoverySnapshotWithoutInterveningPushIsApplied(t *testing.T) {
	old := pb.ServerAddress("10.0.0.1:8888")
	replacement := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(old)

	fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, fc.peerUpdateGeneration())

	if got := filerAddressList(fc); len(got) != 1 || got[0] != replacement {
		t.Fatalf("expected snapshot to replace list, got %v", got)
	}
}

// A leave suppressed to keep the last filer is deferred, then honored as soon
// as a replacement joins, so the departed address stops being a candidate.
func TestOnPeerUpdateDeferredLeaveFlushesOnJoin(t *testing.T) {
	old := pb.ServerAddress("10.0.0.1:8888")
	joined := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(old)

	fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
	if got := filerAddressList(fc); len(got) != 1 || got[0] != old {
		t.Fatalf("leave of the last filer must be deferred, got %v", got)
	}
	if len(fc.deferredLeaves) != 1 {
		t.Fatalf("expected the leave to be deferred, got %v", fc.deferredLeaves)
	}

	fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())

	if got := filerAddressList(fc); len(got) != 1 || got[0] != joined {
		t.Fatalf("deferred leave should flush on join, got %v", got)
	}
	if len(fc.deferredLeaves) != 0 {
		t.Fatalf("deferred leaves should be empty after flush, got %v", fc.deferredLeaves)
	}
}

// A pushed add for an already-known filer changes nothing and must not bump
// the generation that guards an in-flight discovery snapshot.
func TestOnPeerUpdateNoopDoesNotDiscardSnapshot(t *testing.T) {
	old := pb.ServerAddress("10.0.0.1:8888")
	replacement := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(old)

	generation := fc.peerUpdateGeneration()
	fc.OnPeerUpdate(filerUpdate(old, true), time.Now())
	fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, generation)

	if got := filerAddressList(fc); len(got) != 1 || got[0] != replacement {
		t.Fatalf("no-op push should not discard the snapshot, got %v", got)
	}
}

func TestOnPeerUpdateRejoinCancelsDeferredLeave(t *testing.T) {
	restarted := pb.ServerAddress("10.0.0.1:8888")
	joined := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(restarted)

	fc.OnPeerUpdate(filerUpdate(restarted, false), time.Now())
	fc.OnPeerUpdate(filerUpdate(restarted, true), time.Now())
	fc.OnPeerUpdate(filerUpdate(joined, true), time.Now())

	if got := filerAddressList(fc); len(got) != 2 || got[0] != restarted || got[1] != joined {
		t.Fatalf("rejoined filer was dropped by its stale deferred leave: %v", got)
	}
}

func TestDiscoverySnapshotClearsDeferredLeaves(t *testing.T) {
	departed := pb.ServerAddress("10.0.0.1:8888")
	replacement := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(departed)

	fc.OnPeerUpdate(filerUpdate(departed, false), time.Now())
	fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{replacement: {}}, fc.peerUpdateGeneration())
	fc.OnPeerUpdate(filerUpdate(departed, true), time.Now())

	if got := filerAddressList(fc); len(got) != 2 || got[0] != replacement || got[1] != departed {
		t.Fatalf("rejoined filer was dropped by a deferred leave the poll should have cleared: %v", got)
	}
}

// A poll that started after a deferred leave but before the filer rejoined can
// return a snapshot lacking the rejoined filer; the rejoin must bump the
// generation so that snapshot is discarded.
func TestOnPeerUpdateRejoinBumpsGeneration(t *testing.T) {
	old := pb.ServerAddress("10.0.0.1:8888")
	other := pb.ServerAddress("10.0.1.1:8888")
	fc := newTestFilerClient(old)

	fc.OnPeerUpdate(filerUpdate(old, false), time.Now())
	generation := fc.peerUpdateGeneration()
	fc.OnPeerUpdate(filerUpdate(old, true), time.Now())
	fc.applyDiscoverySnapshot(map[pb.ServerAddress]struct{}{other: {}}, generation)

	if got := filerAddressList(fc); len(got) != 1 || got[0] != old {
		t.Fatalf("rejoined filer was pruned by an in-flight snapshot: %v", got)
	}
}

// stallingFiler answers LookupVolume only when the call is abandoned.
type stallingFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
}

func (s *stallingFiler) LookupVolume(ctx context.Context, _ *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
	<-ctx.Done()
	return nil, ctx.Err()
}

func startStallingFiler(t *testing.T) pb.ServerAddress {
	t.Helper()
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(grpcServer, &stallingFiler{})
	go func() { _ = grpcServer.Serve(lis) }()
	t.Cleanup(grpcServer.Stop)

	_, port, _ := net.SplitHostPort(lis.Addr().String())
	return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
}

// lapsedDeadlineCtx mimics the moment after a caller's deadline has passed but
// before the runtime marks the context done.
type lapsedDeadlineCtx struct{ context.Context }

func (lapsedDeadlineCtx) Deadline() (time.Time, bool) { return time.Now().Add(-time.Second), true }
func (lapsedDeadlineCtx) Done() <-chan struct{}       { return nil }
func (lapsedDeadlineCtx) Err() error                  { return nil }

// A caller giving up must not count as a filer failure, the filer's own timeout must.
func TestLookupVolumeIdsCountsOnlyFilerFailures(t *testing.T) {
	tests := []struct {
		name         string
		callerCtx    func() (context.Context, context.CancelFunc)
		grpcTimeout  time.Duration
		wantErr      error
		wantFailures int32
	}{
		{
			name: "caller cancelled",
			callerCtx: func() (context.Context, context.CancelFunc) {
				ctx, cancel := context.WithCancel(context.Background())
				time.AfterFunc(50*time.Millisecond, cancel)
				return ctx, cancel
			},
			grpcTimeout:  5 * time.Second,
			wantErr:      context.Canceled,
			wantFailures: 0,
		},
		{
			name: "caller deadline",
			callerCtx: func() (context.Context, context.CancelFunc) {
				return context.WithTimeout(context.Background(), 50*time.Millisecond)
			},
			grpcTimeout:  5 * time.Second,
			wantErr:      context.DeadlineExceeded,
			wantFailures: 0,
		},
		{
			name: "caller deadline already elapsed",
			callerCtx: func() (context.Context, context.CancelFunc) {
				return lapsedDeadlineCtx{context.Background()}, func() {}
			},
			grpcTimeout:  5 * time.Second,
			wantErr:      context.DeadlineExceeded,
			wantFailures: 0,
		},
		{
			name: "filer timeout",
			callerCtx: func() (context.Context, context.CancelFunc) {
				return context.WithCancel(context.Background())
			},
			grpcTimeout:  50 * time.Millisecond,
			wantFailures: 1,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			fc := newTestFilerClient(startStallingFiler(t))
			fc.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
			fc.grpcTimeout = tt.grpcTimeout
			fc.maxRetries = 1
			fc.failureThreshold = 3
			fc.resetTimeout = time.Minute

			ctx, cancel := tt.callerCtx()
			defer cancel()
			_, err := (&filerVolumeProvider{filerClient: fc}).LookupVolumeIds(ctx, []string{"4"})

			if err == nil {
				t.Fatal("lookup against a stalling filer succeeded")
			}
			if tt.wantErr != nil && !errors.Is(err, tt.wantErr) {
				t.Errorf("err = %v, want %v", err, tt.wantErr)
			}
			if got := atomic.LoadInt32(&fc.filerHealth[0].failureCount); got != tt.wantFailures {
				t.Errorf("failureCount = %d, want %d", got, tt.wantFailures)
			}
		})
	}
}

func TestOnVolumeLocationChangeDropsRetiredServer(t *testing.T) {
	fc := newTestFilerClient()
	retired := Location{Url: "retired:8080"}
	live := Location{Url: "live:8080"}
	fc.addLocation(7, retired)
	fc.addLocation(7, live)
	fc.addLocation(8, retired)

	fc.OnVolumeLocationChange(retired, nil, []uint32{7, 8, 9})

	locations, found := fc.GetLocations(7)
	if !found || len(locations) != 1 || locations[0].Url != live.Url {
		t.Errorf("vid 7 kept only the surviving location: found=%v %v", found, locations)
	}
	if locations, found := fc.GetLocations(8); found && len(locations) > 0 {
		t.Errorf("vid 8 lost its last location but stayed cached: %v", locations)
	}
}

func TestOnVidMapResetClearsCachedLocations(t *testing.T) {
	fc := newTestFilerClient()
	fc.addLocation(7, Location{Url: "live:8080"})

	fc.OnVidMapReset()

	if locations, found := fc.GetLocations(7); found && len(locations) > 0 {
		t.Errorf("vid 7 survived the reset: %v", locations)
	}
}

func TestOnVolumeLocationChangeReaddClearsTombstone(t *testing.T) {
	retired := Location{Url: "retired:8080"}
	fc := newTestFilerClient()
	fc.vidMapClient = newVidMapClient(&testLocationProvider{
		locations: map[string][]Location{"5": {retired}},
	}, "", 0)
	ctx := context.Background()

	fc.OnVolumeLocationChange(retired, nil, []uint32{5})
	if locations, _ := fc.LookupVolumeIdsWithFallback(ctx, []string{"5"}); len(locations["5"]) > 0 {
		t.Fatalf("removed location still served: %v", locations)
	}

	fc.OnVolumeLocationChange(retired, []uint32{5}, nil)
	locations, err := fc.LookupVolumeIdsWithFallback(ctx, []string{"5"})
	if err != nil || len(locations["5"]) != 1 || locations["5"][0].Url != retired.Url {
		t.Fatalf("re-added location not served again: %v, err %v", locations, err)
	}
}

func hookedFilerClient(mc *MasterClient, provider VolumeLocationProvider) *FilerClient {
	fc := newTestFilerClient()
	fc.vidMapClient = newVidMapClient(provider, "", 0)
	mc.SetOnVolumeLocationChangeFn(fc.OnVolumeLocationChange)
	mc.SetOnVidMapResetFn(fc.OnVidMapReset)
	return fc
}

func expectServed(t *testing.T, fc *FilerClient, vid string, want ...Location) {
	t.Helper()
	locations, err := fc.LookupVolumeIdsWithFallback(context.Background(), []string{vid})
	got := locations[vid]
	if len(want) == 0 {
		if len(got) != 0 {
			t.Fatalf("volume %s still served from %v", vid, got)
		}
		return
	}
	if err != nil || len(got) != len(want) {
		t.Fatalf("volume %s served from %v, err %v, want %v", vid, got, err, want)
	}
	for i := range want {
		if got[i].Url != want[i].Url {
			t.Fatalf("volume %s served from %v, want %v", vid, got, want)
		}
	}
}

// ec.decode mounts the regular copy on a shard server before the shards come
// off it, so the later shard removal must leave that copy servable.
func TestDecodedCopyOutlivesShardRemoval(t *testing.T) {
	a := Location{Url: "a:8080"}
	b := Location{Url: "b:8080"}
	provider := &testLocationProvider{locations: map[string][]Location{"5": {a, b}}}
	mc := moveClient()
	fc := hookedFilerClient(mc, provider)
	pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
	pushVolumeLocation(mc, b.Url, nil, nil, []uint32{5}, nil)
	expectServed(t, fc, "5", a, b)

	pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
	pushVolumeLocation(mc, b.Url, nil, nil, nil, []uint32{5})
	pushVolumeLocation(mc, a.Url, nil, nil, nil, []uint32{5})

	provider.locations["5"] = []Location{a, b}
	expectServed(t, fc, "5", a)
}

// ec.encode registers the shards before the regular copy is deleted from the
// same server, so the server stays a read candidate for the ec volume.
func TestShardServerOutlivesRegularRemoval(t *testing.T) {
	a := Location{Url: "a:8080"}
	provider := &testLocationProvider{locations: map[string][]Location{"5": {a}}}
	mc := moveClient()
	fc := hookedFilerClient(mc, provider)
	pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
	expectServed(t, fc, "5", a)

	pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
	pushVolumeLocation(mc, a.Url, nil, []uint32{5}, nil, nil)

	expectServed(t, fc, "5", a)
}

// The filer caches shard servers like regular copies, so the last shard
// leaving a server retires it from the cache too.
func TestRetiredShardServerLeavesCache(t *testing.T) {
	a := Location{Url: "a:8080"}
	b := Location{Url: "b:8080"}
	provider := &testLocationProvider{locations: map[string][]Location{"5": {a, b}}}
	mc := moveClient()
	fc := hookedFilerClient(mc, provider)
	pushVolumeLocation(mc, a.Url, nil, nil, []uint32{5}, nil)
	pushVolumeLocation(mc, b.Url, nil, nil, []uint32{5}, nil)
	expectServed(t, fc, "5", a, b)

	pushVolumeLocation(mc, a.Url, nil, nil, nil, []uint32{5})

	if cached, _ := fc.GetLocations(5); len(cached) != 1 || cached[0].Url != b.Url {
		t.Fatalf("cache holds %v, want only %v", cached, b)
	}
	fc.InvalidateCache("5,01")
	expectServed(t, fc, "5", b)
}

// A volume moved to another server while the old one stays up: the old
// server would proxy the read and succeed, so no failed read would ever
// evict it. The pushed removal has to.
func TestMovedVolumeLeavesOldServer(t *testing.T) {
	old := Location{Url: "old:8080"}
	moved := Location{Url: "new:8080"}
	provider := &testLocationProvider{locations: map[string][]Location{"5": {old}}}
	mc := moveClient()
	fc := hookedFilerClient(mc, provider)
	pushVolumeLocation(mc, old.Url, []uint32{5}, nil, nil, nil)
	expectServed(t, fc, "5", old)

	pushVolumeLocation(mc, moved.Url, []uint32{5}, nil, nil, nil)
	pushVolumeLocation(mc, old.Url, nil, []uint32{5}, nil, nil)

	provider.locations["5"] = []Location{old, moved}
	expectServed(t, fc, "5", moved)
}

// A server back at the same URL after a master reconnect is re-told as an
// add, which lifts the fence its earlier removal left.
func TestReconnectReplayLiftsFence(t *testing.T) {
	a := Location{Url: "a:8080"}
	provider := &testLocationProvider{locations: map[string][]Location{"5": {a}}}
	mc := moveClient()
	fc := hookedFilerClient(mc, provider)
	pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)
	pushVolumeLocation(mc, a.Url, nil, []uint32{5}, nil, nil)
	expectServed(t, fc, "5")

	mc.resetVidMap()
	pushVolumeLocation(mc, a.Url, []uint32{5}, nil, nil, nil)

	expectServed(t, fc, "5", a)
}