mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-21 05:24:28 +02:00
81ed379884
* volume server: route VolumeMarkReadonly to raft leader After a master raft election, volume servers may still heartbeat a follower while admin paths such as weed shell volume.mark call notifyMasterVolumeReadonly via vs.GetMaster(). Followers reject VolumeMarkReadonly with NotLeader, which breaks tiering and other mark-readonly workflows until the heartbeat loop reconnects. Resolve the leader through GetMasterConfiguration on configured -master peers (same Leader field filer/master clients already use) before calling VolumeMarkReadonly. When the leader differs from the heartbeat peer, update currentMaster so the heartbeat loop converges faster. Adds operation.LookupRaftLeaderMaster with unit tests. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address review feedback on volume.mark raft leader routing Do not update currentMaster during leader lookup — heartbeat owns that field and uses stream GetLeader() to reconnect. Try the heartbeat peer first and only resolve the raft leader after a NotLeader rejection. Add ctx.Err() early exit and quieter logging for context cancellation. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(operation): thread the lookup timeout ctx into connection invalidation The 5s timeout drove only the RPC; WithMasterServerClient saw the unbounded outer ctx, so a self-inflicted timeout (slow GetMasterConfiguration during an election) was treated as a stale channel and tore down the shared master connection. Pass the timeout ctx into the helper so its own expiry leaves ctx.Err() set and spares the connection. --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
100 lines
2.7 KiB
Go
100 lines
2.7 KiB
Go
package operation
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
)
|
|
|
|
type fakeLeaderConfigServer struct {
|
|
master_pb.UnimplementedSeaweedServer
|
|
leader string
|
|
}
|
|
|
|
func (s *fakeLeaderConfigServer) GetMasterConfiguration(_ context.Context, _ *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) {
|
|
return &master_pb.GetMasterConfigurationResponse{Leader: s.leader}, nil
|
|
}
|
|
|
|
func startFakeMasterServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
|
|
t.Helper()
|
|
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen: %v", err)
|
|
}
|
|
grpcServer := grpc.NewServer()
|
|
master_pb.RegisterSeaweedServer(grpcServer, srv)
|
|
go func() { _ = grpcServer.Serve(lis) }()
|
|
t.Cleanup(grpcServer.GracefulStop)
|
|
|
|
_, port, _ := net.SplitHostPort(lis.Addr().String())
|
|
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
|
|
}
|
|
|
|
func TestLookupRaftLeaderMasterFromFollower(t *testing.T) {
|
|
follower := startFakeMasterServer(t, &fakeLeaderConfigServer{
|
|
leader: "leader.example.com:9333.19333",
|
|
})
|
|
|
|
leader, err := LookupRaftLeaderMaster(
|
|
context.Background(),
|
|
[]pb.ServerAddress{follower},
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("lookup: %v", err)
|
|
}
|
|
if got, want := leader.ToHttpAddress(), "leader.example.com:9333"; got != want {
|
|
t.Fatalf("leader = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestLookupRaftLeaderMasterTriesNextPeer(t *testing.T) {
|
|
down := pb.ServerAddress("127.0.0.1:1")
|
|
peer := startFakeMasterServer(t, &fakeLeaderConfigServer{
|
|
leader: "leader.example.com:9333.19333",
|
|
})
|
|
|
|
leader, err := LookupRaftLeaderMaster(
|
|
context.Background(),
|
|
[]pb.ServerAddress{down, peer},
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("lookup: %v", err)
|
|
}
|
|
if got, want := leader.ToHttpAddress(), "leader.example.com:9333"; got != want {
|
|
t.Fatalf("leader = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestLookupRaftLeaderMasterNoPeers(t *testing.T) {
|
|
_, err := LookupRaftLeaderMaster(context.Background(), nil, nil)
|
|
if err == nil {
|
|
t.Fatal("expected error for empty peers")
|
|
}
|
|
}
|
|
|
|
func TestLookupRaftLeaderMasterCanceledContext(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
_, err := LookupRaftLeaderMaster(
|
|
ctx,
|
|
[]pb.ServerAddress{"127.0.0.1:1"},
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
|
)
|
|
if err == nil {
|
|
t.Fatal("expected error for canceled context")
|
|
}
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected context.Canceled, got %v", err)
|
|
}
|
|
}
|