mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
ec: scrub a volume through its parity data (#11006)
* Introduce a new `READS` scrub mode. `READS` performs a full volume scrub but, unlike `FULL`, it will attempt to reconstruct data for missing/damaged shard intervals from other shards in the cluster when necessary. The goal of this check is to ensure that EC volume contents _are readable by Seaweed_ even on a degraded storage state, by exercising parity data which is not read in `FULL` mode. This is useful not only to validate data is user-readable, but also to detect potential parity shard issues which may be difficult to pinpoint otherwise - particularly for older volumes lacking sidecar data, and hence unaffected by `CHECKSUM` scrubs. For regular volumes, this operation is equivalent to `FULL`. Example: ``` > ec.shard.unmount --volumeId=1 --shardId=0,3,11 --delete --apply Live shard topology for volume ID 1 (14 shards): 0@10.200.18.89:9001 1@10.200.18.89:9002 2@10.200.18.89:9003 3@10.200.18.89:9004 4@10.200.18.89:9005 5@10.200.18.89:9006 6@10.200.18.89:9007 7@10.200.18.89:9008 8@10.200.18.89:9009 9@10.200.18.89:9013 10@10.200.18.89:9010 11@10.200.18.89:9011 12@10.200.18.89:9012 13@10.200.18.89:9020 Will unmount + delete 3 shard(s): 0@10.200.18.89:9001 3@10.200.18.89:9004 11@10.200.18.89:9011 Unmounting shard 0@10.200.18.89:9001 for volume ID 1... Deleting shard 0@10.200.18.89:9001 for volume ID 1... Unmounting shard 3@10.200.18.89:9004 for volume ID 1... Deleting shard 3@10.200.18.89:9004 for volume ID 1... Unmounting shard 11@10.200.18.89:9011 for volume ID 1... Deleting shard 11@10.200.18.89:9011 for volume ID 1... All done! > ec.scrub --volumeId=1 --node=10.200.18.89:9002 --mode=full using FULL mode Scrubbing 10.200.18.89:9002 (1/1)... Scrubbed 6 EC files and 1 volumes on 1 nodes Got scrub failures on 1 EC volumes and 1 EC shards :( Affected volumes: 10.200.18.89:9002:1 Affected shards: 10.200.18.89:9002:1:0 > ec.scrub --volumeId=1 --node=10.200.18.89:9002 --mode=reads using READS mode Scrubbing 10.200.18.89:9002 (1/1)... Scrubbed 6 EC files and 1 volumes on 1 nodes ``` * ec: report the shards a READS scrub had to rebuild A READS scrub that recovers an interval was recording nothing, so a volume missing three shards came back clean and nobody repaired it. The unreadable shard is now recorded before the rebuild is attempted: READS reports the same broken shards as FULL and differs only in whether the needles themselves failed, which is the signal worth having - shards are gone, data is still there. forceDeletedNeedlesCheck now applies to READS as well, in the shell and in the RPC guard: it runs the same needle walk as FULL. Regenerated the proto instead of hand-editing it, so the pancis typo (which protoc-gen-go-grpc emits into eight other files here) and the header whitespace stay as generated. Mirrors into the Rust volume server, which also now honors force_deleted_needles_check rather than hardcoding it off. Claude-Session: https://claude.ai/code/session_014yMNebkUjSbx9sfUCWJJtq * ec: answer a deleted needle from a READS rebuild as deleted #11020 gave the Rust recovery a deleted flag alongside its bytes, and it answers a deleted needle with no bytes at all. The READS scrub appended that empty answer, which does not compile against the new signature and, once it did, would leave the needle short and report the size mismatch as damage. Zero-fill the interval instead, the way the direct read beside it already does: the assembled needle then reaches read_bytes as the delete-state mismatch the walk already tolerates. Go takes the same branch off the flag its recovery returns, rather than discarding it. Claude-Session: https://claude.ai/code/session_014yMNebkUjSbx9sfUCWJJtq --------- Co-authored-by: Lisandro Pin <lisandro.pin@proton.ch>
This commit is contained in:
1 parent
ba5b14b457
commit
fcc2ea61d3
12 files changed
+375
-109
No files matched your search
@@ -718,6 +718,7 @@ enum VolumeScrubMode {
|
||||
FULL = 2;
|
||||
LOCAL = 3;
|
||||
CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
|
||||
READS = 5; // like FULL, but EC intervals no shard can serve are reconstructed from parity
|
||||
}
|
||||
|
||||
message ScrubVolumeRequest {
|
||||
|
||||
@@ -76,6 +76,7 @@ const (
|
||||
VolumeScrubMode_FULL VolumeScrubMode = 2
|
||||
VolumeScrubMode_LOCAL VolumeScrubMode = 3
|
||||
VolumeScrubMode_CHECKSUM VolumeScrubMode = 4 // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
|
||||
VolumeScrubMode_READS VolumeScrubMode = 5 // like FULL, but EC intervals no shard can serve are reconstructed from parity
|
||||
)
|
||||
|
||||
// Enum value maps for VolumeScrubMode.
|
||||
@@ -86,6 +87,7 @@ var (
|
||||
2: "FULL",
|
||||
3: "LOCAL",
|
||||
4: "CHECKSUM",
|
||||
5: "READS",
|
||||
}
|
||||
VolumeScrubMode_value = map[string]int32{
|
||||
"UNKNOWN": 0,
|
||||
@@ -93,6 +95,7 @@ var (
|
||||
"FULL": 2,
|
||||
"LOCAL": 3,
|
||||
"CHECKSUM": 4,
|
||||
"READS": 5,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -7748,13 +7751,14 @@ const file_volume_server_proto_rawDesc = "" +
|
||||
"stopTimeNs*;\n" +
|
||||
"\x11ChecksumAlgorithm\x12\x11\n" +
|
||||
"\rCHECKSUM_NONE\x10\x00\x12\x13\n" +
|
||||
"\x0fCHECKSUM_CRC32C\x10\x01*L\n" +
|
||||
"\x0fCHECKSUM_CRC32C\x10\x01*W\n" +
|
||||
"\x0fVolumeScrubMode\x12\v\n" +
|
||||
"\aUNKNOWN\x10\x00\x12\t\n" +
|
||||
"\x05INDEX\x10\x01\x12\b\n" +
|
||||
"\x04FULL\x10\x02\x12\t\n" +
|
||||
"\x05LOCAL\x10\x03\x12\f\n" +
|
||||
"\bCHECKSUM\x10\x042\xfa)\n" +
|
||||
"\bCHECKSUM\x10\x04\x12\t\n" +
|
||||
"\x05READS\x10\x052\xfa)\n" +
|
||||
"\fVolumeServer\x12\\\n" +
|
||||
"\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" +
|
||||
"\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" +
|
||||
|
||||
@@ -43,8 +43,9 @@ func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.S
|
||||
switch m := req.GetMode(); m {
|
||||
case volume_server_pb.VolumeScrubMode_INDEX:
|
||||
files, serrs = v.ScrubIndex()
|
||||
case volume_server_pb.VolumeScrubMode_LOCAL:
|
||||
// LOCAL is equivalent to FULL for regular volumes
|
||||
case volume_server_pb.VolumeScrubMode_LOCAL, volume_server_pb.VolumeScrubMode_READS:
|
||||
// both are equivalent to FULL for regular volumes: there are no shards to
|
||||
// stay local to, and nothing to reconstruct from
|
||||
fallthrough
|
||||
case volume_server_pb.VolumeScrubMode_FULL:
|
||||
files, serrs = v.Scrub()
|
||||
@@ -96,8 +97,9 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
|
||||
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if req.GetForceDeletedNeedlesCheck() && req.GetMode() != volume_server_pb.VolumeScrubMode_FULL {
|
||||
return nil, fmt.Errorf("deleted needle checks are only supported for FULL scrubs")
|
||||
if m := req.GetMode(); req.GetForceDeletedNeedlesCheck() &&
|
||||
m != volume_server_pb.VolumeScrubMode_FULL && m != volume_server_pb.VolumeScrubMode_READS {
|
||||
return nil, fmt.Errorf("deleted needle checks are only supported for FULL and READS scrubs")
|
||||
}
|
||||
|
||||
vids := []needle.VolumeId{}
|
||||
@@ -130,8 +132,8 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
|
||||
files, serrs = v.ScrubIndex()
|
||||
case volume_server_pb.VolumeScrubMode_LOCAL:
|
||||
files, shardInfos, serrs = v.ScrubLocal()
|
||||
case volume_server_pb.VolumeScrubMode_FULL:
|
||||
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId, req.GetForceDeletedNeedlesCheck())
|
||||
case volume_server_pb.VolumeScrubMode_FULL, volume_server_pb.VolumeScrubMode_READS:
|
||||
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId, m, req.GetForceDeletedNeedlesCheck())
|
||||
case volume_server_pb.VolumeScrubMode_CHECKSUM:
|
||||
// Verify each local shard's raw bytes against the bitrot sidecar,
|
||||
// exercising cold parity shards. Read-only. ChecksumScrub's first
|
||||
|
||||
@@ -41,10 +41,10 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
|
||||
volScrubCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
nodesStr := volScrubCommand.String("node", "", "comma-separated list of volume server <host>:<port> (optional)")
|
||||
volumeIDsStr := volScrubCommand.String("volumeId", "", "comma-separated EC volume IDs to process (optional)")
|
||||
mode := volScrubCommand.String("mode", "local", "scrubbing mode (index/local/full/checksum)")
|
||||
mode := volScrubCommand.String("mode", "local", "scrubbing mode (index/local/full/reads/checksum)")
|
||||
maxParallelization := volScrubCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
|
||||
showDetails := volScrubCommand.Bool("details", false, "display scrub result details, if available")
|
||||
forceDeletedNeedlesCheck := volScrubCommand.Bool("forceDeletedNeedlesCheck", false, "force strict verification of deleted needles (full mode only); may report false positives when EC indexes disagree")
|
||||
forceDeletedNeedlesCheck := volScrubCommand.Bool("forceDeletedNeedlesCheck", false, "force strict verification of deleted needles (full and reads modes only); may report false positives when EC indexes disagree")
|
||||
|
||||
if err = volScrubCommand.Parse(args); err != nil {
|
||||
return err
|
||||
@@ -97,12 +97,15 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
|
||||
scrubMode = volume_server_pb.VolumeScrubMode_FULL
|
||||
case "CHECKSUM":
|
||||
scrubMode = volume_server_pb.VolumeScrubMode_CHECKSUM
|
||||
case "READS":
|
||||
scrubMode = volume_server_pb.VolumeScrubMode_READS
|
||||
default:
|
||||
return fmt.Errorf("unsupported scrubbing mode %q", *mode)
|
||||
}
|
||||
fmt.Fprintf(writer, "using %s mode\n", scrubMode.String())
|
||||
if *forceDeletedNeedlesCheck && scrubMode != volume_server_pb.VolumeScrubMode_FULL {
|
||||
return fmt.Errorf("deleted needle checks are only supported for FULL scrubs")
|
||||
if *forceDeletedNeedlesCheck &&
|
||||
scrubMode != volume_server_pb.VolumeScrubMode_FULL && scrubMode != volume_server_pb.VolumeScrubMode_READS {
|
||||
return fmt.Errorf("deleted needle checks are only supported for FULL and READS scrubs")
|
||||
}
|
||||
|
||||
return ec.ScrubEcVolumes(commandEnv.ecEnv(), writer, volumeServerAddrs, volumeIDs, scrubMode, *forceDeletedNeedlesCheck, *maxParallelization, *showDetails)
|
||||
|
||||
@@ -50,7 +50,7 @@ func (c *commandVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer io
|
||||
volScrubCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
nodesStr := volScrubCommand.String("node", "", "comma-separated list of volume server <host>:<port> (optional)")
|
||||
volumeIDsStr := volScrubCommand.String("volumeId", "", "comma-separated volume IDs to process (optional)")
|
||||
mode := volScrubCommand.String("mode", "full", "scrubbing mode (index/local/full)")
|
||||
mode := volScrubCommand.String("mode", "full", "scrubbing mode (index/local/full/reads)")
|
||||
markBrokenReadonly := volScrubCommand.Bool("markBrokenReadonly", false, "whether to flag volumes with scrub failures as read-only")
|
||||
maxParallelization := volScrubCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
|
||||
showDetails := volScrubCommand.Bool("details", false, "display scrub result details, if available")
|
||||
@@ -99,6 +99,9 @@ func (c *commandVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer io
|
||||
c.mode = volume_server_pb.VolumeScrubMode_LOCAL
|
||||
case "FULL":
|
||||
c.mode = volume_server_pb.VolumeScrubMode_FULL
|
||||
case "READS":
|
||||
// only meaningful for EC volumes; accepted here so one mode name works on both commands
|
||||
c.mode = volume_server_pb.VolumeScrubMode_READS
|
||||
default:
|
||||
return fmt.Errorf("unsupported scrubbing mode %q", *mode)
|
||||
}
|
||||
|
||||
@@ -2,90 +2,17 @@ package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
)
|
||||
|
||||
// A needle larger than one EC block is split over consecutive blocks, which
|
||||
// live on different shards. The intervals are read concurrently, so check they
|
||||
// still come back in order.
|
||||
func TestReadEcShardNeedleSpanningBlocks(t *testing.T) {
|
||||
store := newTestStore(t, 1)
|
||||
dir := store.Locations[0].Directory
|
||||
const vid = needle.VolumeId(7)
|
||||
|
||||
v, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("new volume: %v", err)
|
||||
}
|
||||
n := new(needle.Needle)
|
||||
n.Id = types.Uint64ToNeedleId(42)
|
||||
n.Data = make([]byte, 3*erasure_coding.ErasureCodingSmallBlockSize+1234)
|
||||
rand.New(rand.NewSource(42)).Read(n.Data)
|
||||
n.Checksum = needle.NewCRC(n.Data)
|
||||
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
||||
t.Fatalf("write needle: %v", err)
|
||||
}
|
||||
baseFileName := v.DataFileName()
|
||||
v.Close()
|
||||
|
||||
datSize, err := os.Stat(baseFileName + ".dat")
|
||||
if err != nil {
|
||||
t.Fatalf("stat .dat: %v", err)
|
||||
}
|
||||
if _, err := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()); err != nil {
|
||||
t.Fatalf("write ec files: %v", err)
|
||||
}
|
||||
if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
|
||||
t.Fatalf("write .ecx: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(baseFileName+".ecj", nil, 0o644); err != nil {
|
||||
t.Fatalf("write .ecj: %v", err)
|
||||
}
|
||||
if err := volume_info.SaveVolumeInfo(baseFileName+".vif", &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
DatFileSize: datSize.Size(),
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{
|
||||
DataShards: erasure_coding.DataShardsCount,
|
||||
ParityShards: erasure_coding.ParityShardsCount,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("save .vif: %v", err)
|
||||
}
|
||||
for _, ext := range []string{".dat", ".idx"} {
|
||||
if err := os.Remove(baseFileName + ext); err != nil {
|
||||
t.Fatalf("remove %s: %v", ext, err)
|
||||
}
|
||||
}
|
||||
|
||||
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
|
||||
if err := store.MountEcShards("", vid, erasure_coding.ShardId(shardId), ""); err != nil {
|
||||
t.Fatalf("mount shard %d: %v", shardId, err)
|
||||
}
|
||||
}
|
||||
ecVolume, found := store.Locations[0].FindEcVolume(vid)
|
||||
if !found {
|
||||
t.Fatal("ec volume not mounted")
|
||||
}
|
||||
|
||||
// Every shard is local, so seed the location cache to keep the read off the
|
||||
// master this test does not have.
|
||||
ecVolume.ShardLocationsLock.Lock()
|
||||
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
|
||||
ecVolume.ShardLocations[erasure_coding.ShardId(shardId)] = []pb.ServerAddress{"localhost:8080"}
|
||||
}
|
||||
ecVolume.ShardLocationsRefreshTime = time.Now()
|
||||
ecVolume.ShardLocationsLock.Unlock()
|
||||
store, ecVolume, n := newLocalEcVolume(t, vid)
|
||||
|
||||
_, _, intervals, err := ecVolume.LocateEcShardNeedle(n.Id, ecVolume.Version)
|
||||
if err != nil {
|
||||
|
||||
@@ -13,7 +13,11 @@ import (
|
||||
|
||||
// ScrubEcVolume checks the full integrity of a EC volume, across both local and remote shards.
|
||||
// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
|
||||
func (s *Store) ScrubEcVolume(vid needle.VolumeId, forceDeletedNeedlesCheck bool) (int64, []*volume_server_pb.EcShardInfo, []error) {
|
||||
//
|
||||
// FULL reports an unreadable shard and gives up on the needle. READS additionally rebuilds the
|
||||
// interval from the surviving shards, so it reports the same broken shards but only errors on
|
||||
// needles that parity can no longer recover - which is what exercises the parity data.
|
||||
func (s *Store) ScrubEcVolume(vid needle.VolumeId, mode volume_server_pb.VolumeScrubMode, forceDeletedNeedlesCheck bool) (int64, []*volume_server_pb.EcShardInfo, []error) {
|
||||
ecv, found := s.FindEcVolume(vid)
|
||||
if !found {
|
||||
return 0, nil, []error{fmt.Errorf("EC volume id %d not found", vid)}
|
||||
@@ -25,6 +29,8 @@ func (s *Store) ScrubEcVolume(vid needle.VolumeId, forceDeletedNeedlesCheck bool
|
||||
// full scan means verifying indexes as well
|
||||
_, errs := ecv.ScrubIndex()
|
||||
|
||||
recoverUnreadable := mode == volume_server_pb.VolumeScrubMode_READS
|
||||
|
||||
var count int64
|
||||
// reads for EC chunks can hit the same shard multiple times, so dedupe upon read errors
|
||||
brokenShardsMap := map[erasure_coding.ShardId]*volume_server_pb.EcShardInfo{}
|
||||
@@ -56,27 +62,39 @@ func (s *Store) ScrubEcVolume(vid needle.VolumeId, forceDeletedNeedlesCheck bool
|
||||
continue
|
||||
}
|
||||
|
||||
// ...then remote. note we do not try to recover EC-encoded data upon read failures;
|
||||
// we want check that shards are valid without decoding
|
||||
// ...then remote. neither read decodes: the point is to find shards that are
|
||||
// themselves broken, not to heal around them
|
||||
ecv.ShardLocationsLock.RLock()
|
||||
sourceDataNodes, ok := ecv.ShardLocations[shardId]
|
||||
ecv.ShardLocationsLock.RUnlock()
|
||||
readErr := errors.New("no known shard locations")
|
||||
if ok {
|
||||
if _, _, err := s.readRemoteEcShardInterval(sourceDataNodes, id, ecv.VolumeId, shardId, chunk, offset, ecv.EncodeTsNs); err == nil {
|
||||
if _, _, readErr = s.readRemoteEcShardInterval(sourceDataNodes, id, ecv.VolumeId, shardId, chunk, offset, ecv.EncodeTsNs); readErr == nil {
|
||||
data = append(data, chunk...)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// chunk read for shard failed :(
|
||||
errs = append(errs, fmt.Errorf("failed to read EC shard %d for needle %d on volume %d (interval %d/%d)", shardId, id, ecv.VolumeId, i+1, len(intervals)))
|
||||
// the shard is broken whether or not the needle survives it, so report it either way
|
||||
brokenShardsMap[shardId] = &volume_server_pb.EcShardInfo{
|
||||
ShardId: uint32(shardId),
|
||||
Size: int64(iv.Size),
|
||||
Collection: ecv.Collection,
|
||||
VolumeId: uint32(ecv.VolumeId),
|
||||
}
|
||||
break
|
||||
|
||||
if !recoverUnreadable {
|
||||
errs = append(errs, fmt.Errorf("failed to read EC shard %d for needle %d on volume %d (interval %d/%d): %v", shardId, id, ecv.VolumeId, i+1, len(intervals), readErr))
|
||||
break
|
||||
}
|
||||
// A holder reporting the needle deleted is authoritative, and it answers
|
||||
// with no bytes: the chunk stays zeroed and reaches ReadBytes as the
|
||||
// delete-state mismatch the walk below already tolerates.
|
||||
if _, isDeleted, err := s.recoverOneRemoteEcShardInterval(id, ecv, shardId, chunk, offset); err != nil && !isDeleted {
|
||||
errs = append(errs, fmt.Errorf("failed to recover EC shard %d for needle %d on volume %d (interval %d/%d): %v", shardId, id, ecv.VolumeId, i+1, len(intervals), err))
|
||||
break
|
||||
}
|
||||
data = append(data, chunk...)
|
||||
}
|
||||
|
||||
if got, want := int64(len(data)), needle.GetActualSize(size, ecv.Version); got != want {
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
)
|
||||
|
||||
// newLocalEcVolume builds an EC volume holding one needle spread over several shards,
|
||||
// mounts every shard locally, and seeds the shard-location cache with empty location
|
||||
// lists: the cache then counts as fresh, so nothing calls the master these tests do not
|
||||
// have, and no read leaves the process. Returns the needle that was encoded.
|
||||
func newLocalEcVolume(t *testing.T, vid needle.VolumeId) (*Store, *erasure_coding.EcVolume, *needle.Needle) {
|
||||
t.Helper()
|
||||
|
||||
store := newTestStore(t, 1)
|
||||
dir := store.Locations[0].Directory
|
||||
|
||||
v, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("new volume: %v", err)
|
||||
}
|
||||
n := new(needle.Needle)
|
||||
n.Id = types.Uint64ToNeedleId(42)
|
||||
n.Data = make([]byte, 3*erasure_coding.ErasureCodingSmallBlockSize+1234)
|
||||
rand.New(rand.NewSource(42)).Read(n.Data)
|
||||
n.Checksum = needle.NewCRC(n.Data)
|
||||
if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
|
||||
t.Fatalf("write needle: %v", err)
|
||||
}
|
||||
baseFileName := v.DataFileName()
|
||||
v.Close()
|
||||
|
||||
datSize, err := os.Stat(baseFileName + ".dat")
|
||||
if err != nil {
|
||||
t.Fatalf("stat .dat: %v", err)
|
||||
}
|
||||
if _, err := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()); err != nil {
|
||||
t.Fatalf("write ec files: %v", err)
|
||||
}
|
||||
if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
|
||||
t.Fatalf("write .ecx: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(baseFileName+".ecj", nil, 0o644); err != nil {
|
||||
t.Fatalf("write .ecj: %v", err)
|
||||
}
|
||||
if err := volume_info.SaveVolumeInfo(baseFileName+".vif", &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
DatFileSize: datSize.Size(),
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{
|
||||
DataShards: erasure_coding.DataShardsCount,
|
||||
ParityShards: erasure_coding.ParityShardsCount,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("save .vif: %v", err)
|
||||
}
|
||||
for _, ext := range []string{".dat", ".idx"} {
|
||||
if err := os.Remove(baseFileName + ext); err != nil {
|
||||
t.Fatalf("remove %s: %v", ext, err)
|
||||
}
|
||||
}
|
||||
|
||||
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
|
||||
if err := store.MountEcShards("", vid, erasure_coding.ShardId(shardId), ""); err != nil {
|
||||
t.Fatalf("mount shard %d: %v", shardId, err)
|
||||
}
|
||||
}
|
||||
ecVolume, found := store.Locations[0].FindEcVolume(vid)
|
||||
if !found {
|
||||
t.Fatal("ec volume not mounted")
|
||||
}
|
||||
|
||||
ecVolume.ShardLocationsLock.Lock()
|
||||
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
|
||||
ecVolume.ShardLocations[erasure_coding.ShardId(shardId)] = []pb.ServerAddress{}
|
||||
}
|
||||
ecVolume.ShardLocationsRefreshTime = time.Now()
|
||||
ecVolume.ShardLocationsLock.Unlock()
|
||||
|
||||
return store, ecVolume, n
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeReadsMatchesFullWhenHealthy(t *testing.T) {
|
||||
const vid = needle.VolumeId(7)
|
||||
store, _, _ := newLocalEcVolume(t, vid)
|
||||
|
||||
for _, mode := range []volume_server_pb.VolumeScrubMode{
|
||||
volume_server_pb.VolumeScrubMode_FULL,
|
||||
volume_server_pb.VolumeScrubMode_READS,
|
||||
} {
|
||||
count, brokenShards, errs := store.ScrubEcVolume(vid, mode, false)
|
||||
if count != 1 {
|
||||
t.Errorf("%s: scrubbed %d needles, want 1", mode, count)
|
||||
}
|
||||
if len(brokenShards) != 0 {
|
||||
t.Errorf("%s: reported broken shards %v on a healthy volume", mode, brokenShards)
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
t.Errorf("%s: reported errors on a healthy volume: %v", mode, errs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A READS scrub rebuilds what it cannot read, but the shard it could not read is still
|
||||
// broken and must still be reported - otherwise a volume missing shards scrubs clean and
|
||||
// the operator never learns to repair it.
|
||||
func TestScrubEcVolumeReadsReportsTheShardItRebuilt(t *testing.T) {
|
||||
const vid = needle.VolumeId(7)
|
||||
const missingShard = erasure_coding.ShardId(0)
|
||||
|
||||
store, _, _ := newLocalEcVolume(t, vid)
|
||||
if err := store.UnmountEcShards(vid, missingShard, 0); err != nil {
|
||||
t.Fatalf("unmount shard %d: %v", missingShard, err)
|
||||
}
|
||||
|
||||
// FULL gives up on the needles that lived on the missing shard...
|
||||
_, brokenShards, errs := store.ScrubEcVolume(vid, volume_server_pb.VolumeScrubMode_FULL, false)
|
||||
assertOnlyBrokenShard(t, "FULL", brokenShards, missingShard)
|
||||
if len(errs) == 0 {
|
||||
t.Fatalf("FULL reported no error for a shard it could not read")
|
||||
}
|
||||
if got := errs[0].Error(); !strings.Contains(got, "failed to read EC shard 0") {
|
||||
t.Fatalf("FULL error %q, want it to name the shard it could not read", got)
|
||||
}
|
||||
|
||||
// ...READS rebuilds them from the shards that are left, and still names the shard.
|
||||
_, brokenShards, errs = store.ScrubEcVolume(vid, volume_server_pb.VolumeScrubMode_READS, false)
|
||||
assertOnlyBrokenShard(t, "READS", brokenShards, missingShard)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("READS should rebuild every needle from parity, got: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeReadsErrorsWhenParityCannotCover(t *testing.T) {
|
||||
const vid = needle.VolumeId(7)
|
||||
|
||||
store, _, _ := newLocalEcVolume(t, vid)
|
||||
// one shard more than parity can cover, so nothing can be rebuilt
|
||||
for shardId := 0; shardId <= erasure_coding.ParityShardsCount; shardId++ {
|
||||
if err := store.UnmountEcShards(vid, erasure_coding.ShardId(shardId), 0); err != nil {
|
||||
t.Fatalf("unmount shard %d: %v", shardId, err)
|
||||
}
|
||||
}
|
||||
|
||||
_, brokenShards, errs := store.ScrubEcVolume(vid, volume_server_pb.VolumeScrubMode_READS, false)
|
||||
if len(brokenShards) == 0 {
|
||||
t.Fatalf("no broken shard reported for an unrecoverable volume")
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
t.Fatalf("no error reported for an unrecoverable volume")
|
||||
}
|
||||
if got := errs[0].Error(); !strings.Contains(got, "failed to recover EC shard") {
|
||||
t.Fatalf("error %q, want it to say the rebuild failed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func assertOnlyBrokenShard(t *testing.T, mode string, brokenShards []*volume_server_pb.EcShardInfo, shardId erasure_coding.ShardId) {
|
||||
t.Helper()
|
||||
if len(brokenShards) != 1 || brokenShards[0].GetShardId() != uint32(shardId) {
|
||||
t.Fatalf("%s reported broken shards %v, want only shard %d", mode, brokenShards, shardId)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user