mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-20 04:54:24 +02:00
81ed379884
* volume server: route VolumeMarkReadonly to raft leader After a master raft election, volume servers may still heartbeat a follower while admin paths such as weed shell volume.mark call notifyMasterVolumeReadonly via vs.GetMaster(). Followers reject VolumeMarkReadonly with NotLeader, which breaks tiering and other mark-readonly workflows until the heartbeat loop reconnects. Resolve the leader through GetMasterConfiguration on configured -master peers (same Leader field filer/master clients already use) before calling VolumeMarkReadonly. When the leader differs from the heartbeat peer, update currentMaster so the heartbeat loop converges faster. Adds operation.LookupRaftLeaderMaster with unit tests. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address review feedback on volume.mark raft leader routing Do not update currentMaster during leader lookup — heartbeat owns that field and uses stream GetLeader() to reconnect. Try the heartbeat peer first and only resolve the raft leader after a NotLeader rejection. Add ctx.Err() early exit and quieter logging for context cancellation. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(operation): thread the lookup timeout ctx into connection invalidation The 5s timeout drove only the RPC; WithMasterServerClient saw the unbounded outer ctx, so a self-inflicted timeout (slow GetMasterConfiguration during an election) was treated as a stale channel and tore down the shared master connection. Pass the timeout ctx into the helper so its own expiry leaves ctx.Err() set and spares the connection. --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
55 lines
1.9 KiB
Go
55 lines
1.9 KiB
Go
package operation
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
// LookupRaftLeaderMaster returns the current raft leader by querying configured
|
|
// master peers. Followers answer GetMasterConfiguration with the leader address;
|
|
// topology mutations such as VolumeMarkReadonly must run on that node.
|
|
//
|
|
// The volume server's heartbeat peer (GetMaster) may still be a follower after an
|
|
// election until the heartbeat loop reconnects.
|
|
func LookupRaftLeaderMaster(ctx context.Context, masters []pb.ServerAddress, grpcDialOption grpc.DialOption) (pb.ServerAddress, error) {
|
|
if len(masters) == 0 {
|
|
return "", fmt.Errorf("no master peers configured")
|
|
}
|
|
var lastErr error
|
|
for _, peer := range masters {
|
|
if err := ctx.Err(); err != nil {
|
|
return "", err
|
|
}
|
|
// The same timeout ctx drives both the RPC and WithMasterServerClient's
|
|
// connection-invalidation decision, so a self-inflicted 5s timeout leaves
|
|
// callCtx.Err() != nil and does not poison the shared master connection.
|
|
callCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
var leader pb.ServerAddress
|
|
err := WithMasterServerClient(callCtx, false, peer, grpcDialOption, func(client master_pb.SeaweedClient) error {
|
|
resp, err := client.GetMasterConfiguration(callCtx, &master_pb.GetMasterConfigurationRequest{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.Leader == "" {
|
|
return fmt.Errorf("master %s returned empty leader", peer)
|
|
}
|
|
leader = pb.ServerAddress(resp.Leader)
|
|
return nil
|
|
})
|
|
cancel()
|
|
if err == nil && leader != "" {
|
|
return leader, nil
|
|
}
|
|
lastErr = err
|
|
}
|
|
if lastErr != nil {
|
|
return "", fmt.Errorf("lookup raft leader from %d master peer(s): %w", len(masters), lastErr)
|
|
}
|
|
return "", fmt.Errorf("lookup raft leader: no leader from %d master peer(s)", len(masters))
|
|
}
|