volume: let evacuation proceed on a server in maintenance mode (#11145)

Maintenance mode exists to fence a volume server so it can be evacuated
without taking new writes (#7977), but the gate added in #8115 also
rejected the RPCs evacuation issues against the source: VolumeMarkReadonly
(the first step of every move, and the failure reported in #11066),
VolumeDelete (the last step), and VolumeEcShardsDelete (the last step for
EC shards). volumeServer.evacuate, volume.move and ec.balance therefore
all failed on exactly the server they were meant to drain.

Those three RPCs only remove data or restrict the server further, the same
class as DeleteCollection and the unmount RPCs that were never gated, so
they are exempted from the maintenance check in both the Go and Rust
volume servers. Everything that adds data or reopens the server for
writes (AllocateVolume, WriteNeedleBlob, BatchDelete, VolumeCopy,
ReceiveFile, EC generate/copy/rebuild, vacuum, tiering, VolumeMarkWritable)
stays blocked. A side effect is that scrub can now fence broken volumes
readonly on a server already in maintenance.

Fixes #11066

Generated with [Devin](https://devin.ai)

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

weed/server/volume_grpc_admin.go

@@ -186,6 +186,9 @@ func (vs *VolumeServer) VolumeConsolidateIndex(ctx context.Context, req *volume_
 
 }
 
+// VolumeDelete is allowed in maintenance mode: it removes data from the server
+// rather than adding any, and evacuating a server in maintenance mode ends each
+// move by deleting the source copy (issue #11066).
 func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
 	resp := &volume_server_pb.VolumeDeleteResponse{}
 
@@ -193,10 +196,6 @@ func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.
 		return resp, err
 	}
 
-	if err := vs.CheckMaintenanceMode(); err != nil {
-		return resp, err
-	}
-
 	err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId), req.OnlyEmpty, req.KeepRemoteData)
 
 	if err != nil {
@@ -271,11 +270,10 @@ func (vs *VolumeServer) VolumeConfigure(ctx context.Context, req *volume_server_
 
 }
 
+// makeVolumeReadonly is not gated on maintenance mode: marking a volume readonly
+// only restricts a server that is already meant to be read-only, and it is the
+// first step of moving a volume off a server under evacuation (issue #11066).
 func (vs *VolumeServer) makeVolumeReadonly(ctx context.Context, v *storage.Volume, canDelete bool, persist bool) error {
-	if err := vs.CheckMaintenanceMode(); err != nil {
-		return err
-	}
-
 	// step 1: stop master from redirecting traffic here
 	if err := vs.notifyMasterVolumeReadonly(ctx, v, true); err != nil {
 		return err

weed/server/volume_grpc_erasure_coding.go

@@ -464,13 +464,12 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 
 // VolumeEcShardsDelete local delete the .ecx and some ec data slices if not needed
 // the shard should not be mounted before calling this.
+// Allowed in maintenance mode: like VolumeDelete it only removes data, and
+// evacuating EC shards off a server in maintenance mode ends here (issue #11066).
 func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
 	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
 		return nil, err
 	}
-	if err := vs.CheckMaintenanceMode(); err != nil {
-		return nil, err
-	}
 
 	bName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId))
 

weed/server/volume_grpc_maintenance_test.go

@@ -0,0 +1,111 @@
+package weed_server
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newMaintenanceModeServer builds a volume server whose store holds volume vid
+// in collection and has maintenance mode switched on.
+func newMaintenanceModeServer(t *testing.T, vid needle.VolumeId, collection string) (*VolumeServer, string) {
+	t.Helper()
+	dir := t.TempDir()
+	store := newTraversalTestStore(dir)
+	t.Cleanup(store.Close)
+	require.NoError(t, store.AddVolume(vid, collection, storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
+	require.NoError(t, store.State.Update(&volume_server_pb.VolumeServerState{Maintenance: true}))
+
+	vs := &VolumeServer{
+		store:          store,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+	require.True(t, vs.MaintenanceMode())
+	return vs, dir
+}
+
+type fakeReadonlyAcceptingMaster struct {
+	master_pb.UnimplementedSeaweedServer
+}
+
+func (s *fakeReadonlyAcceptingMaster) VolumeMarkReadonly(context.Context, *master_pb.VolumeMarkReadonlyRequest) (*master_pb.VolumeMarkReadonlyResponse, error) {
+	return &master_pb.VolumeMarkReadonlyResponse{}, nil
+}
+
+// Evacuating a server in maintenance mode marks each volume readonly on the
+// source, copies it, then deletes the source (issue #11066). Those source-side
+// RPCs remove data or restrict the server further, so maintenance mode must
+// let them through — otherwise the mode defeats the evacuation it exists for.
+func TestMaintenanceModeAllowsVolumeMarkReadonly(t *testing.T) {
+	vid := needle.VolumeId(1)
+	vs, _ := newMaintenanceModeServer(t, vid, "")
+	vs.setCurrentMaster(startFakeMasterServerForLeaderLookup(t, &fakeReadonlyAcceptingMaster{}))
+
+	_, err := vs.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: uint32(vid)})
+	require.NoError(t, err)
+	assert.True(t, vs.store.GetVolume(vid).IsReadOnly())
+}
+
+func TestMaintenanceModeAllowsVolumeDelete(t *testing.T) {
+	vid := needle.VolumeId(2)
+	vs, _ := newMaintenanceModeServer(t, vid, "")
+
+	_, err := vs.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{VolumeId: uint32(vid)})
+	require.NoError(t, err)
+	assert.Nil(t, vs.store.GetVolume(vid), "volume should be gone from the store")
+}
+
+func TestMaintenanceModeAllowsVolumeEcShardsDelete(t *testing.T) {
+	const collection = "ec-maint"
+	vid := needle.VolumeId(3)
+	vs, dir := newMaintenanceModeServer(t, needle.VolumeId(99), "")
+
+	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
+	require.NoError(t, os.WriteFile(base+".ecx", make([]byte, 16), 0o644))
+	for _, id := range []int{0, 1} {
+		require.NoError(t, os.WriteFile(base+erasure_coding.ToExt(id), []byte("s"), 0o644))
+	}
+
+	_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   uint32(vid),
+		Collection: collection,
+		ShardIds:   []uint32{0, 1},
+	})
+	require.NoError(t, err)
+	assert.False(t, util.FileExists(base+erasure_coding.ToExt(0)))
+	assert.False(t, util.FileExists(base+erasure_coding.ToExt(1)))
+}
+
+// Maintenance mode keeps rejecting RPCs that add data to the server or reopen
+// it for writes; only the removal/restriction path above is exempt.
+func TestMaintenanceModeStillBlocksWrites(t *testing.T) {
+	vid := needle.VolumeId(4)
+	vs, _ := newMaintenanceModeServer(t, vid, "")
+	wantErr := vs.CheckMaintenanceMode().Error()
+	ctx := context.Background()
+
+	_, err := vs.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 5, Replication: "000", DiskType: string(types.HardDriveType)})
+	assert.EqualError(t, err, wantErr, "AllocateVolume")
+
+	_, err = vs.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{VolumeId: uint32(vid), NeedleId: 1, Size: 1, NeedleBlob: []byte{0}})
+	assert.EqualError(t, err, wantErr, "WriteNeedleBlob")
+
+	_, err = vs.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"4,01637037d6"}})
+	assert.EqualError(t, err, wantErr, "BatchDelete")
+
+	_, err = vs.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: uint32(vid)})
+	assert.EqualError(t, err, wantErr, "VolumeMarkWritable")
+}