diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index cb49e7f3e..586bf37f4 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -152,9 +152,10 @@ func (ms *MasterServer) ProcessGrowRequest() { // we have lock called inside vg glog.V(0).Infof("volume grow %+v", req) go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) { + // defer so a panic can't strand growRequest. + defer filter.Delete(req) + defer vl.DoneGrowRequest() ms.DoAutomaticVolumeGrow(req) - vl.DoneGrowRequest() - filter.Delete(req) }(req, vl) } }() diff --git a/weed/topology/allocate_volume.go b/weed/topology/allocate_volume.go index ab869c184..ef1d844e4 100644 --- a/weed/topology/allocate_volume.go +++ b/weed/topology/allocate_volume.go @@ -2,6 +2,7 @@ package topology import ( "context" + "time" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -9,6 +10,10 @@ import ( "google.golang.org/grpc" ) +// Cap the RPC so a hung volume server can't strand the layout's +// growRequest flag and block all future automatic growth. +const allocateVolumeTimeout = 1 * time.Minute + type AllocateVolumeResult struct { Error string } @@ -17,7 +22,10 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, allocateErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{ + ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout) + defer cancel() + + _, allocateErr := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{ VolumeId: uint32(vid), Collection: option.Collection, Replication: option.ReplicaPlacement.String(), @@ -36,7 +44,10 @@ func DeleteVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Volum return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, allocateErr := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ + ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout) + defer cancel() + + _, allocateErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{ VolumeId: uint32(vid), }) return allocateErr