mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* wdclient: let master-pushed adds replace the gateway's lookup-fed locations After a master leader change the S3 gateway clears its FilerClient cache and refills it through filer lookups. The filer's own master client may still answer from the previous leader's generation until the new leader has heard from the volume servers, so a volume server lost together with the old leader is cached again. The new leader never knew that server and never pushes its removal, and the gateway keeps trying it until reads fail on it volume by volume, one 30 s dial timeout at a time. The gateway's master client relearns every volume from the new leader's adds. On each add, cache the volume where the master client now knows it to be, in place of what a lookup cached, and keep lookups already in flight from caching their answer over it. A provider answer for a volume is now cached whole or not at all. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: take a lookup's fill epoch before its second cache check A master add that landed between the second cache check and the read of the fill epoch was invisible to both: the check had already missed, and the epoch the lookup then read already included the confirmation, so a filer answer that still named the lost server was merged back into the confirmed entry. Taking the epoch first means such an add either shows up in the check or stops the answer from being cached. The same order covers a reset or clear in that gap. TestConfirmDuringLookupIsNotUndone fires a confirmation from a test hook at that point and fails with the old order. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: gate the master lookup provider's cache writes on the map epoch The provider wrote each answer straight into the vid map, so a lookup sent before a master reset and answered after it landed in the new generation, which a gateway's FilerClient now reads as the current master's view. The answer now goes through fillLocations, which drops it once a reset passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: keep the vidMapClient doc comment on its type Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
157 lines
4.7 KiB
Go
157 lines
4.7 KiB
Go
package wdclient
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// fakeLookupServer returns Unavailable for the first N calls, then succeeds.
|
|
type fakeLookupServer struct {
|
|
master_pb.UnimplementedSeaweedServer
|
|
unavailableCount int32
|
|
callCount atomic.Int32
|
|
}
|
|
|
|
func (s *fakeLookupServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
|
|
n := s.callCount.Add(1)
|
|
if n <= s.unavailableCount {
|
|
return nil, status.Errorf(codes.Unavailable, "master is warming up")
|
|
}
|
|
resp := &master_pb.LookupVolumeResponse{}
|
|
for _, vid := range req.VolumeOrFileIds {
|
|
resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
|
|
VolumeOrFileId: vid,
|
|
Locations: []*master_pb.Location{
|
|
{Url: "127.0.0.1:8080", PublicUrl: "127.0.0.1:8080"},
|
|
},
|
|
})
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
func startFakeMasterServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
|
|
t.Helper()
|
|
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
grpcServer := grpc.NewServer()
|
|
master_pb.RegisterSeaweedServer(grpcServer, srv)
|
|
go func() { _ = grpcServer.Serve(lis) }()
|
|
t.Cleanup(grpcServer.GracefulStop)
|
|
|
|
_, port, _ := net.SplitHostPort(lis.Addr().String())
|
|
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
|
|
}
|
|
|
|
func TestLookupVolumeIdsRetriesOnUnavailable(t *testing.T) {
|
|
srv := &fakeLookupServer{unavailableCount: 3}
|
|
addr := startFakeMasterServer(t, srv)
|
|
|
|
mc := NewMasterClient(
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
"", "test", "", "", "",
|
|
pb.ServerDiscovery{},
|
|
)
|
|
mc.setCurrentMaster(addr)
|
|
mc.grpcTimeout = 5 * time.Second
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
provider := &masterVolumeProvider{masterClient: mc}
|
|
result, err := provider.LookupVolumeIds(ctx, []string{"1"})
|
|
|
|
if err != nil {
|
|
t.Fatalf("expected success after retries, got error: %v", err)
|
|
}
|
|
if _, ok := result["1"]; !ok {
|
|
t.Error("expected volume 1 in result")
|
|
}
|
|
if calls := srv.callCount.Load(); calls != 4 {
|
|
t.Errorf("expected 4 calls (3 unavailable + 1 success), got %d", calls)
|
|
}
|
|
}
|
|
|
|
func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) {
|
|
srv := &fakeLookupServer{unavailableCount: 1000}
|
|
addr := startFakeMasterServer(t, srv)
|
|
|
|
mc := NewMasterClient(
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
"", "test", "", "", "",
|
|
pb.ServerDiscovery{},
|
|
)
|
|
mc.setCurrentMaster(addr)
|
|
mc.grpcTimeout = 5 * time.Second
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
provider := &masterVolumeProvider{masterClient: mc}
|
|
start := time.Now()
|
|
_, err := provider.LookupVolumeIds(ctx, []string{"1"})
|
|
elapsed := time.Since(start)
|
|
|
|
// Verify the error is from context deadline
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("expected context.DeadlineExceeded, got: %v", err)
|
|
}
|
|
// Verify the loop actually retried (not just an immediate failure)
|
|
if calls := srv.callCount.Load(); calls <= 1 {
|
|
t.Errorf("expected multiple retry attempts, got %d calls", calls)
|
|
}
|
|
if elapsed > 5*time.Second {
|
|
t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed)
|
|
}
|
|
}
|
|
|
|
type resetDuringLookupServer struct {
|
|
fakeLookupServer
|
|
mc *MasterClient
|
|
}
|
|
|
|
func (s *resetDuringLookupServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
|
|
s.mc.resetVidMap()
|
|
return s.fakeLookupServer.LookupVolume(ctx, req)
|
|
}
|
|
|
|
// An answer sent before a reset must not land in the new generation, which a
|
|
// gateway's FilerClient reads as the current master's view.
|
|
func TestLookupAnswerCrossingResetIsNotCached(t *testing.T) {
|
|
mc := NewMasterClient(
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
"", "test", "", "", "",
|
|
pb.ServerDiscovery{},
|
|
)
|
|
srv := &resetDuringLookupServer{mc: mc}
|
|
mc.setCurrentMaster(startFakeMasterServer(t, srv))
|
|
mc.grpcTimeout = 5 * time.Second
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
provider := &masterVolumeProvider{masterClient: mc}
|
|
result, err := provider.LookupVolumeIds(ctx, []string{"1"})
|
|
if err != nil {
|
|
t.Fatalf("lookup: %v", err)
|
|
}
|
|
if _, ok := result["1"]; !ok {
|
|
t.Fatal("expected volume 1 to be served")
|
|
}
|
|
if locations, found := mc.vidMap.currentLocations(1); found {
|
|
t.Fatalf("answer that crossed a reset was cached: %v", locations)
|
|
}
|
|
}
|