master: timeout AllocateVolume/DeleteVolume and defer growRequest cleanup (#9698)

* master: timeout AllocateVolume/DeleteVolume and defer growRequest cleanup

The volume-grow goroutine clears the layout's growRequest flag only after
ms.DoAutomaticVolumeGrow returns, and AllocateVolume / DeleteVolume were
calling the volume-server RPC with context.Background(). A volume server
that hung mid-call (heavy I/O, stuck lock, dead peer behind a stable VIP)
would park the goroutine forever, leaving growRequest=true and silently
blocking every subsequent automatic grow for that layout — Assign retries
then drained their 30s budget with "context deadline exceeded" until the
operator restarted the master.

Bound both RPCs with a 5-minute deadline (creating/removing a volume is
sub-second normally, generous for contended disks) and move the flag
clear + filter delete into defers so a panic in DoAutomaticVolumeGrow
doesn't strand the layout either.

* allocate_volume: shorten timeout to 1m for faster recovery

Volume create/delete is sub-second under normal conditions; 1 minute is
generous even on a contended disk and clears the growRequest flag well
before too many client Assigns drain their own retry budget.

* trim comments

weed/server/master_grpc_server_volume.go

@@ -152,9 +152,10 @@ func (ms *MasterServer) ProcessGrowRequest() {
 			// we have lock called inside vg
 			glog.V(0).Infof("volume grow %+v", req)
 			go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) {
+				// defer so a panic can't strand growRequest.
+				defer filter.Delete(req)
+				defer vl.DoneGrowRequest()
 				ms.DoAutomaticVolumeGrow(req)
-				vl.DoneGrowRequest()
-				filter.Delete(req)
 			}(req, vl)
 		}
 	}()

weed/topology/allocate_volume.go

@@ -2,6 +2,7 @@ package topology
 
 import (
 	"context"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -9,6 +10,10 @@ import (
 	"google.golang.org/grpc"
 )
 
+// Cap the RPC so a hung volume server can't strand the layout's
+// growRequest flag and block all future automatic growth.
+const allocateVolumeTimeout = 1 * time.Minute
+
 type AllocateVolumeResult struct {
 	Error string
 }
@@ -17,7 +22,10 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol
 
 	return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
-		_, allocateErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{
+		ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout)
+		defer cancel()
+
+		_, allocateErr := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
 			VolumeId:           uint32(vid),
 			Collection:         option.Collection,
 			Replication:        option.ReplicaPlacement.String(),
@@ -36,7 +44,10 @@ func DeleteVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Volum
 
 	return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
-		_, allocateErr := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{
+		ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout)
+		defer cancel()
+
+		_, allocateErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
 			VolumeId: uint32(vid),
 		})
 		return allocateErr