Files
seaweedfs/weed/wdclient/masterclient_retry_test.go
T
97f49571a1 wdclient: let master-pushed adds replace the gateway's lookup-fed locations (#12705)
* wdclient: let master-pushed adds replace the gateway's lookup-fed locations

After a master leader change the S3 gateway clears its FilerClient cache and
refills it through filer lookups. The filer's own master client may still
answer from the previous leader's generation until the new leader has heard
from the volume servers, so a volume server lost together with the old leader
is cached again. The new leader never knew that server and never pushes its
removal, and the gateway keeps trying it until reads fail on it volume by
volume, one 30 s dial timeout at a time.

The gateway's master client relearns every volume from the new leader's
adds. On each add, cache the volume where the master client now knows it to
be, in place of what a lookup cached, and keep lookups already in flight from
caching their answer over it. A provider answer for a volume is now cached
whole or not at all.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: take a lookup's fill epoch before its second cache check

A master add that landed between the second cache check and the read of
the fill epoch was invisible to both: the check had already missed, and
the epoch the lookup then read already included the confirmation, so a
filer answer that still named the lost server was merged back into the
confirmed entry. Taking the epoch first means such an add either shows
up in the check or stops the answer from being cached. The same order
covers a reset or clear in that gap.

TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook
at that point and fails with the old order.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: gate the master lookup provider's cache writes on the map epoch

The provider wrote each answer straight into the vid map, so a lookup sent
before a master reset and answered after it landed in the new generation,
which a gateway's FilerClient now reads as the current master's view. The
answer now goes through fillLocations, which drops it once a reset passed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: keep the vidMapClient doc comment on its type

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2026-11-17 19:10:20 -08:00

157 lines
4.7 KiB
Go

package wdclient
import (
"context"
"errors"
"fmt"
"net"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
// fakeLookupServer returns Unavailable for the first N calls, then succeeds.
type fakeLookupServer struct {
master_pb.UnimplementedSeaweedServer
unavailableCount int32
callCount atomic.Int32
}
func (s *fakeLookupServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
n := s.callCount.Add(1)
if n <= s.unavailableCount {
return nil, status.Errorf(codes.Unavailable, "master is warming up")
}
resp := &master_pb.LookupVolumeResponse{}
for _, vid := range req.VolumeOrFileIds {
resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
VolumeOrFileId: vid,
Locations: []*master_pb.Location{
{Url: "127.0.0.1:8080", PublicUrl: "127.0.0.1:8080"},
},
})
}
return resp, nil
}
func startFakeMasterServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
grpcServer := grpc.NewServer()
master_pb.RegisterSeaweedServer(grpcServer, srv)
go func() { _ = grpcServer.Serve(lis) }()
t.Cleanup(grpcServer.GracefulStop)
_, port, _ := net.SplitHostPort(lis.Addr().String())
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
}
func TestLookupVolumeIdsRetriesOnUnavailable(t *testing.T) {
srv := &fakeLookupServer{unavailableCount: 3}
addr := startFakeMasterServer(t, srv)
mc := NewMasterClient(
grpc.WithTransportCredentials(insecure.NewCredentials()),
"", "test", "", "", "",
pb.ServerDiscovery{},
)
mc.setCurrentMaster(addr)
mc.grpcTimeout = 5 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
provider := &masterVolumeProvider{masterClient: mc}
result, err := provider.LookupVolumeIds(ctx, []string{"1"})
if err != nil {
t.Fatalf("expected success after retries, got error: %v", err)
}
if _, ok := result["1"]; !ok {
t.Error("expected volume 1 in result")
}
if calls := srv.callCount.Load(); calls != 4 {
t.Errorf("expected 4 calls (3 unavailable + 1 success), got %d", calls)
}
}
func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) {
srv := &fakeLookupServer{unavailableCount: 1000}
addr := startFakeMasterServer(t, srv)
mc := NewMasterClient(
grpc.WithTransportCredentials(insecure.NewCredentials()),
"", "test", "", "", "",
pb.ServerDiscovery{},
)
mc.setCurrentMaster(addr)
mc.grpcTimeout = 5 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
provider := &masterVolumeProvider{masterClient: mc}
start := time.Now()
_, err := provider.LookupVolumeIds(ctx, []string{"1"})
elapsed := time.Since(start)
// Verify the error is from context deadline
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("expected context.DeadlineExceeded, got: %v", err)
}
// Verify the loop actually retried (not just an immediate failure)
if calls := srv.callCount.Load(); calls <= 1 {
t.Errorf("expected multiple retry attempts, got %d calls", calls)
}
if elapsed > 5*time.Second {
t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed)
}
}
type resetDuringLookupServer struct {
fakeLookupServer
mc *MasterClient
}
func (s *resetDuringLookupServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
s.mc.resetVidMap()
return s.fakeLookupServer.LookupVolume(ctx, req)
}
// An answer sent before a reset must not land in the new generation, which a
// gateway's FilerClient reads as the current master's view.
func TestLookupAnswerCrossingResetIsNotCached(t *testing.T) {
mc := NewMasterClient(
grpc.WithTransportCredentials(insecure.NewCredentials()),
"", "test", "", "", "",
pb.ServerDiscovery{},
)
srv := &resetDuringLookupServer{mc: mc}
mc.setCurrentMaster(startFakeMasterServer(t, srv))
mc.grpcTimeout = 5 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
provider := &masterVolumeProvider{masterClient: mc}
result, err := provider.LookupVolumeIds(ctx, []string{"1"})
if err != nil {
t.Fatalf("lookup: %v", err)
}
if _, ok := result["1"]; !ok {
t.Fatal("expected volume 1 to be served")
}
if locations, found := mc.vidMap.currentLocations(1); found {
t.Fatalf("answer that crossed a reset was cached: %v", locations)
}
}