mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-13 01:50:40 +02:00
Maintenance mode exists to fence a volume server so it can be evacuated without taking new writes (#7977), but the gate added in #8115 also rejected the RPCs evacuation issues against the source: VolumeMarkReadonly (the first step of every move, and the failure reported in #11066), VolumeDelete (the last step), and VolumeEcShardsDelete (the last step for EC shards). volumeServer.evacuate, volume.move and ec.balance therefore all failed on exactly the server they were meant to drain. Those three RPCs only remove data or restrict the server further, the same class as DeleteCollection and the unmount RPCs that were never gated, so they are exempted from the maintenance check in both the Go and Rust volume servers. Everything that adds data or reopens the server for writes (AllocateVolume, WriteNeedleBlob, BatchDelete, VolumeCopy, ReceiveFile, EC generate/copy/rebuild, vacuum, tiering, VolumeMarkWritable) stays blocked. A side effect is that scrub can now fence broken volumes readonly on a server already in maintenance. Fixes #11066 Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
192 lines
6.8 KiB
Go
192 lines
6.8 KiB
Go
package volume_server_grpc_test
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
|
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
)
|
|
|
|
func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skipping integration test in short mode")
|
|
}
|
|
|
|
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
|
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
|
defer conn.Close()
|
|
|
|
const volumeID = uint32(72)
|
|
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
|
|
VolumeId: volumeID,
|
|
Persist: false,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("VolumeMarkReadonly failed: %v", err)
|
|
}
|
|
|
|
readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeStatus after readonly failed: %v", err)
|
|
}
|
|
if !readOnlyStatus.GetIsReadOnly() {
|
|
t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly")
|
|
}
|
|
|
|
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeMarkWritable failed: %v", err)
|
|
}
|
|
|
|
writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeStatus after writable failed: %v", err)
|
|
}
|
|
if writableStatus.GetIsReadOnly() {
|
|
t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable")
|
|
}
|
|
}
|
|
|
|
func TestVolumeMarkReadonlyPersistTrue(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skipping integration test in short mode")
|
|
}
|
|
|
|
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
|
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
|
defer conn.Close()
|
|
|
|
const volumeID = uint32(74)
|
|
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
|
|
VolumeId: volumeID,
|
|
Persist: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err)
|
|
}
|
|
|
|
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeStatus after persist readonly failed: %v", err)
|
|
}
|
|
if !statusResp.GetIsReadOnly() {
|
|
t.Fatalf("VolumeStatus expected readonly=true after persist readonly")
|
|
}
|
|
}
|
|
|
|
func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skipping integration test in short mode")
|
|
}
|
|
|
|
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
|
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
|
defer conn.Close()
|
|
|
|
const volumeID = uint32(75)
|
|
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
// non-existent volumes should return "not found"
|
|
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true})
|
|
if err == nil || !strings.Contains(err.Error(), "not found") {
|
|
t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err)
|
|
}
|
|
|
|
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772})
|
|
if err == nil || !strings.Contains(err.Error(), "not found") {
|
|
t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err)
|
|
}
|
|
|
|
// enter maintenance mode
|
|
framework.EnableMaintenanceMode(t, ctx, grpcClient)
|
|
|
|
// marking readonly only restricts the server further and is the first step
|
|
// of evacuating it, so maintenance mode lets it through (issue #11066)
|
|
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true})
|
|
if err != nil {
|
|
t.Fatalf("VolumeMarkReadonly should succeed in maintenance mode, got: %v", err)
|
|
}
|
|
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeStatus after readonly in maintenance failed: %v", err)
|
|
}
|
|
if !statusResp.GetIsReadOnly() {
|
|
t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly in maintenance mode")
|
|
}
|
|
|
|
// reopening a volume for writes is still refused on a read-only server
|
|
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
|
|
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
|
|
t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
|
|
}
|
|
|
|
// non-existent volume in maintenance mode should still return "not found"
|
|
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98773, Persist: true})
|
|
if err == nil || !strings.Contains(err.Error(), "not found") {
|
|
t.Fatalf("VolumeMarkReadonly missing-volume in maintenance error mismatch: %v", err)
|
|
}
|
|
|
|
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98774})
|
|
if err == nil || !strings.Contains(err.Error(), "not found") {
|
|
t.Fatalf("VolumeMarkWritable missing-volume in maintenance error mismatch: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skipping integration test in short mode")
|
|
}
|
|
|
|
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
|
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
|
defer conn.Close()
|
|
|
|
const volumeID = uint32(73)
|
|
const collection = "it-delete-collection"
|
|
|
|
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err != nil {
|
|
t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err)
|
|
}
|
|
|
|
_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
|
|
if err != nil {
|
|
t.Fatalf("DeleteCollection existing collection failed: %v", err)
|
|
}
|
|
|
|
_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
|
if err == nil {
|
|
t.Fatalf("VolumeStatus should fail after collection delete")
|
|
}
|
|
if !strings.Contains(err.Error(), "not found volume") {
|
|
t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err)
|
|
}
|
|
|
|
_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
|
|
if err != nil {
|
|
t.Fatalf("DeleteCollection idempotent retry failed: %v", err)
|
|
}
|
|
}
|