From 29eec2f111667867b2f481327b5ac62e7ecc4a26 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 26 May 2026 16:26:21 -0700 Subject: [PATCH] master: timeout AllocateVolume/DeleteVolume and defer growRequest cleanup (#9698) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * master: timeout AllocateVolume/DeleteVolume and defer growRequest cleanup The volume-grow goroutine clears the layout's growRequest flag only after ms.DoAutomaticVolumeGrow returns, and AllocateVolume / DeleteVolume were calling the volume-server RPC with context.Background(). A volume server that hung mid-call (heavy I/O, stuck lock, dead peer behind a stable VIP) would park the goroutine forever, leaving growRequest=true and silently blocking every subsequent automatic grow for that layout — Assign retries then drained their 30s budget with "context deadline exceeded" until the operator restarted the master. Bound both RPCs with a 5-minute deadline (creating/removing a volume is sub-second normally, generous for contended disks) and move the flag clear + filter delete into defers so a panic in DoAutomaticVolumeGrow doesn't strand the layout either. * allocate_volume: shorten timeout to 1m for faster recovery Volume create/delete is sub-second under normal conditions; 1 minute is generous even on a contended disk and clears the growRequest flag well before too many client Assigns drain their own retry budget. * trim comments --- weed/server/master_grpc_server_volume.go | 5 +++-- weed/topology/allocate_volume.go | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index cb49e7f3e..586bf37f4 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -152,9 +152,10 @@ func (ms *MasterServer) ProcessGrowRequest() { // we have lock called inside vg glog.V(0).Infof("volume grow %+v", req) go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) { + // defer so a panic can't strand growRequest. + defer filter.Delete(req) + defer vl.DoneGrowRequest() ms.DoAutomaticVolumeGrow(req) - vl.DoneGrowRequest() - filter.Delete(req) }(req, vl) } }() diff --git a/weed/topology/allocate_volume.go b/weed/topology/allocate_volume.go index ab869c184..ef1d844e4 100644 --- a/weed/topology/allocate_volume.go +++ b/weed/topology/allocate_volume.go @@ -2,6 +2,7 @@ package topology import ( "context" + "time" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -9,6 +10,10 @@ import ( "google.golang.org/grpc" ) +// Cap the RPC so a hung volume server can't strand the layout's +// growRequest flag and block all future automatic growth. +const allocateVolumeTimeout = 1 * time.Minute + type AllocateVolumeResult struct { Error string } @@ -17,7 +22,10 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, allocateErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{ + ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout) + defer cancel() + + _, allocateErr := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ VolumeId: uint32(vid), Collection: option.Collection, Replication: option.ReplicaPlacement.String(), @@ -36,7 +44,10 @@ func DeleteVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Volum return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, allocateErr := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ + ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout) + defer cancel() + + _, allocateErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{ VolumeId: uint32(vid), }) return allocateErr