Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-08 15:41:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

volume: let evacuation proceed on a server in maintenance mode (#11145)

Browse Source 
Maintenance mode exists to fence a volume server so it can be evacuated
without taking new writes (#7977), but the gate added in #8115 also
rejected the RPCs evacuation issues against the source: VolumeMarkReadonly
(the first step of every move, and the failure reported in #11066),
VolumeDelete (the last step), and VolumeEcShardsDelete (the last step for
EC shards). volumeServer.evacuate, volume.move and ec.balance therefore
all failed on exactly the server they were meant to drain.

Those three RPCs only remove data or restrict the server further, the same
class as DeleteCollection and the unmount RPCs that were never gated, so
they are exempted from the maintenance check in both the Go and Rust
volume servers. Everything that adds data or reopens the server for
writes (AllocateVolume, WriteNeedleBlob, BatchDelete, VolumeCopy,
ReceiveFile, EC generate/copy/rebuild, vacuum, tiering, VolumeMarkWritable)
stays blocked. A side effect is that scrub can now fence broken volumes
readonly on a server already in maintenance.

Fixes #11066

Generated with [Devin](https://devin.ai)

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Chris Lu
2026-09-03 18:39:45 -07:00
committed by GitHub
co-authored by Devin
parent c9c6e6fb1d
commit 24b8646ec3
8 changed files with 198 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/volume_server/grpc/admin_lifecycle_test.go
+9 -15
View File
@@ -179,7 +179,9 @@ func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
}
}
func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
// Evacuating a server in maintenance mode ends each move by deleting the source
// volume, so VolumeDelete must stay available in that mode (issue #11066).
func TestMaintenanceModeAllowsVolumeDelete(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
@@ -194,22 +196,14 @@ func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
framework.EnableMaintenanceMode(t, ctx, client)
if _, err := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
t.Fatalf("VolumeDelete should succeed in maintenance mode, got: %v", err)
}
_, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
_, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err == nil {
t.Fatalf("VolumeDelete should fail when maintenance mode is enabled")
}
if !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("expected maintenance mode error, got: %v", err)
t.Fatalf("VolumeStatus should fail once the volume is deleted")
}
}
test/volume_server/grpc/admin_readonly_collection_test.go
+12 -4
View File
@@ -117,19 +117,27 @@ func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
// enter maintenance mode
framework.EnableMaintenanceMode(t, ctx, grpcClient)
// existing volume in maintenance mode should return "maintenance mode" error
// marking readonly only restricts the server further and is the first step
// of evacuating it, so maintenance mode lets it through (issue #11066)
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err)
if err != nil {
t.Fatalf("VolumeMarkReadonly should succeed in maintenance mode, got: %v", err)
}
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus after readonly in maintenance failed: %v", err)
}
if !statusResp.GetIsReadOnly() {
t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly in maintenance mode")
}
// reopening a volume for writes is still refused on a read-only server
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
}
// non-existent volume in maintenance mode should still return "not found"
// (volume lookup happens before maintenance check)
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98773, Persist: true})
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("VolumeMarkReadonly missing-volume in maintenance error mismatch: %v", err)
test/volume_server/grpc/erasure_coding_test.go
+30 -17
View File
@@ -27,24 +27,27 @@ func TestEcMaintenanceModeRejections(t *testing.T) {
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
// Encode a volume before entering maintenance mode so the shard delete
// below has real shards to remove.
const ecVolumeID = uint32(1)
framework.AllocateVolume(t, grpcClient, ecVolumeID, "")
httpClient := framework.NewHTTPClient()
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(ecVolumeID, 990001, 0x11223344), []byte("ec-maintenance-content"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{
Maintenance: true,
Version: stateResp.GetState().GetVersion(),
},
})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: ecVolumeID, Collection: ""}); err != nil {
t.Fatalf("VolumeEcShardsGenerate before maintenance failed: %v", err)
}
_, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
framework.EnableMaintenanceMode(t, ctx, grpcClient)
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err)
}
@@ -59,13 +62,23 @@ func TestEcMaintenanceModeRejections(t *testing.T) {
t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err)
}
// Deleting shards removes data rather than adding it, and is how evacuating
// EC shards off a server in maintenance mode finishes (issue #11066).
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: 1,
VolumeId: ecVolumeID,
Collection: "",
ShardIds: []uint32{0},
})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err)
if err != nil {
t.Fatalf("VolumeEcShardsDelete should succeed in maintenance mode, got: %v", err)
}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: ecVolumeID,
Collection: "",
ShardIds: []uint32{0},
})
if err == nil {
t.Fatalf("VolumeEcShardsMount should fail once shard 0 has been deleted")
}
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
test/volume_server/grpc/scrub_query_test.go
+16 -5
View File
@@ -520,14 +520,25 @@ func TestScrubVolumeMarkBrokenReadonlyInMaintenanceMode(t *testing.T) {
framework.EnableMaintenanceMode(t, ctx, grpcClient)
// scrub with the flag in maintenance mode: makeVolumeReadonly should fail
// and ScrubVolume should propagate the error
_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
// marking a volume readonly only restricts a server that is already
// read-only, so maintenance mode does not block fencing broken volumes
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
MarkBrokenVolumesReadonly: true,
})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly in maintenance mode error mismatch: %v", err)
if err != nil {
t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly in maintenance mode failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after corruption")
}
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus after scrub in maintenance mode failed: %v", err)
}
if !statusResp.GetIsReadOnly() {
t.Fatalf("broken volume should be read-only after MarkBrokenVolumesReadonly scrub in maintenance mode")
}
}