mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 01:37:10 +01:00
ec: mount shards through the encode's .vif when a source volume's .vif shares the disk (#12676)
* ec: mount shards against the index .vif when the local one has no EC config A worker encode can send shards to a disk that still holds the source volume's .vif while the encode's .vif lands with the index on a sibling disk. The shard disk mounted through the source's config-free .vif, so it reported encode time 0 and fell back to the sidecar layout, and reads carrying the encode time were rejected as a different encode run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): mount shards against the index .vif when the local one has no EC config Mirrors the Go mount: a data-dir .vif that records no EC config yields to the idx dir's when that one records it, and load_vif_info_across_dirs reports the directory the .vif actually came from. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: do not mirror EC sidecars onto a disk whose .vif has no EC config Startup mirroring copied .ecx/.ecj next to a source volume's leftover config-free .vif and kept that .vif, so after a restart the shard disk mounted self-contained with encode time 0. Leave such a disk on the cross-disk mount, which reads the owner's .vif. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): do not mirror EC sidecars onto a disk whose .vif has no EC config Mirrors the Go startup mirror: a shard disk holding a source volume's config-free .vif stays on the cross-disk mount, which reads the owner's .vif, instead of mounting self-contained with encode time 0. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec: take the encode .vif from a sibling data dir when the index dir is shared With a separate -dir.idx the .ecx is shared by every disk, but the encode's .vif lands in the data dir of the disk that received it. A disk still holding a source volume's config-free .vif mounted through it at generation 0. NewEcVolume now accepts the server's data dirs and takes a sibling's EC .vif when the index is not local and every sibling EC .vif names the same run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: give EC mounts the data dir of every disk on the server NewStore records every data dir on each DiskLocation before loading, so the startup scan, cross-disk reconcile and MountEcShards all pass them to NewEcVolume. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the encode .vif from a sibling data dir when the index dir is shared Mirrors the Go mount: EcVolume::new_with_vif_dirs takes a sibling's EC .vif when this disk's own records no EC config, the index is not local, and every sibling EC .vif names the same run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): give EC mounts the data dir of every disk on the server main sets Store.store_dirs before the first add_location, so a disk loaded before its siblings still sees them; each DiskLocation passes them to EcVolume::new_with_vif_dirs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: size startup EC validation by the sibling encode .vif With a shared -dir.idx, a disk that still holds the source .dat and its config-free .vif read the default data-shard count, judged custom-ratio shards an interrupted encode, and removed them along with the shared .ecx before the mount could take the sibling's .vif. The startup lookup now falls back to the same sibling .vif the mount uses. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): size startup EC validation by the sibling encode .vif Mirrors the Go change: validate_ec_volume falls back to the sibling data dir's encode .vif before judging shards beside a source .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: keep EC shards when sibling .vif files disagree on the encode SiblingEcConfigVif reports when a sibling .vif cannot be read or the siblings name different encode runs. The startup data-shard lookup then returns 0 instead of the default ratio, and validateEcVolume keeps the files rather than sizing custom-ratio shards by a ratio they do not use. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): keep EC shards when sibling .vif files disagree on the encode Mirrors the Go change: sibling_ec_config_vif returns Err for an unreadable sibling .vif or disagreeing runs, ec_data_shards_from_vif returns 0 then, and validate_ec_volume keeps the files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: release the opened EC shard when its volume cannot be built Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9e5357782e
commit
b449301a4e
13 files changed
+1092
-22
No files matched your search
@@ -171,14 +171,63 @@ func statEcxSize(path string) (int64, error) {
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
|
||||
return newEcVolumeWith(diskType, dir, dirIdx, collection, vid, realEcjFsOps)
|
||||
func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, vifDirs ...string) (ev *EcVolume, err error) {
|
||||
return newEcVolumeWith(diskType, dir, dirIdx, collection, vid, realEcjFsOps, vifDirs...)
|
||||
}
|
||||
|
||||
// preferEcConfigVif returns alt when local records no EC config but alt does.
|
||||
// A source volume's own .vif stays on a disk that later receives shards of the
|
||||
// same volume id, while the encode's .vif lands with the index on a sibling.
|
||||
func preferEcConfigVif(local, alt string) string {
|
||||
localInfo, _, found, err := volume_info.MaybeLoadVolumeInfo(local)
|
||||
if err != nil || (found && localInfo.EcShardConfig != nil) {
|
||||
return local
|
||||
}
|
||||
altInfo, _, altFound, altErr := volume_info.MaybeLoadVolumeInfo(alt)
|
||||
if altErr != nil || !altFound || altInfo.EcShardConfig == nil {
|
||||
return local
|
||||
}
|
||||
return alt
|
||||
}
|
||||
|
||||
// SiblingEcConfigVif returns the encode's .vif that a disk mounting against a
|
||||
// shared -dir.idx takes from another data dir in dirs: dir holds no non-empty
|
||||
// .ecx, dirIdx is not a data dir, and every sibling .vif recording an EC config
|
||||
// names the same encode run. It returns "" otherwise, with ambiguous set when
|
||||
// a sibling .vif cannot be read or the siblings name different encode runs.
|
||||
func SiblingEcConfigVif(collection string, vid needle.VolumeId, dir, dirIdx string, dirs []string) (vif string, ambiguous bool) {
|
||||
if dirIdx == dir || slices.Contains(dirs, dirIdx) {
|
||||
return "", false
|
||||
}
|
||||
if size, err := statEcxSize(EcShardFileName(collection, dir, int(vid)) + ".ecx"); err == nil && size > 0 {
|
||||
return "", false
|
||||
}
|
||||
found, foundTsNs := "", int64(0)
|
||||
for _, d := range dirs {
|
||||
if d == dir {
|
||||
continue
|
||||
}
|
||||
vif := EcShardFileName(collection, d, int(vid)) + ".vif"
|
||||
info, _, ok, err := volume_info.MaybeLoadVolumeInfo(vif)
|
||||
if err != nil {
|
||||
return "", true
|
||||
}
|
||||
if !ok || info.EcShardConfig == nil {
|
||||
continue
|
||||
}
|
||||
if found == "" {
|
||||
found, foundTsNs = vif, info.EcShardConfig.EncodeTsNs
|
||||
} else if info.EcShardConfig.EncodeTsNs != foundTsNs {
|
||||
return "", true
|
||||
}
|
||||
}
|
||||
return found, false
|
||||
}
|
||||
|
||||
// newEcVolumeWith is NewEcVolume with the journal-compaction filesystem steps
|
||||
// supplied, so tests can drive a failing load or publish through the real
|
||||
// mount.
|
||||
func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, ecjOps ecjFsOps) (ev *EcVolume, err error) {
|
||||
func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId, ecjOps ecjFsOps, vifDirs ...string) (ev *EcVolume, err error) {
|
||||
ev = &EcVolume{dir: dir, dirIdx: dirIdx, Collection: collection, VolumeId: vid, diskType: diskType}
|
||||
|
||||
dataBaseFileName := EcShardFileName(collection, dir, int(vid))
|
||||
@@ -312,6 +361,15 @@ func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collect
|
||||
}
|
||||
}
|
||||
}
|
||||
if dirIdx != dir && vifFileName == dataBaseFileName+".vif" {
|
||||
vifFileName = preferEcConfigVif(vifFileName, EcShardFileName(collection, dirIdx, int(vid))+".vif")
|
||||
// A shared -dir.idx holds no .vif: the encode's sits in a sibling data dir.
|
||||
if vifFileName == dataBaseFileName+".vif" {
|
||||
if alt, _ := SiblingEcConfigVif(collection, vid, dir, dirIdx, vifDirs); alt != "" {
|
||||
vifFileName = preferEcConfigVif(vifFileName, alt)
|
||||
}
|
||||
}
|
||||
}
|
||||
ev.Version = needle.Version3
|
||||
// A present-but-unreadable or malformed .vif FAILS the mount: every new
|
||||
// encode records a positive uniform block size there, and defaulting to
|
||||
|
||||
@@ -185,3 +185,131 @@ func TestNewEcVolumeRejectsUnusableSoleSidecar(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shards placed on the disk that still holds the source volume's .vif mount
|
||||
// against an index on a sibling disk; the encode identity and layout come from
|
||||
// the sibling's .vif, not from the source's config-free one.
|
||||
func TestNewEcVolumePrefersIndexVifOverConfigFreeLocalVif(t *testing.T) {
|
||||
dataDir, idxDir := t.TempDir(), t.TempDir()
|
||||
const encodeTsNs = int64(1_700_000_000_123_456_789)
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "c_1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
ReadOnly: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed source .vif: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(idxDir, "c_1.ecx"), []byte{}, 0644); err != nil {
|
||||
t.Fatalf("seed .ecx: %v", err)
|
||||
}
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "c_1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
DatFileSize: 4096,
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{
|
||||
DataShards: 12, ParityShards: 4, BlockSize: 3 * 1024 * 1024, EncodeTsNs: encodeTsNs,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed index .vif: %v", err)
|
||||
}
|
||||
|
||||
ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "c", needle.VolumeId(1))
|
||||
if err != nil {
|
||||
t.Fatalf("mount: %v", err)
|
||||
}
|
||||
defer ev.Close()
|
||||
if ev.EncodeTsNs != encodeTsNs {
|
||||
t.Errorf("EncodeTsNs = %d, want %d", ev.EncodeTsNs, encodeTsNs)
|
||||
}
|
||||
if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 {
|
||||
t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize)
|
||||
}
|
||||
if ev.datFileSize != 4096 {
|
||||
t.Errorf("datFileSize = %d, want 4096", ev.datFileSize)
|
||||
}
|
||||
}
|
||||
|
||||
// A local .vif that records an EC config stays authoritative for its disk.
|
||||
func TestNewEcVolumeKeepsLocalVifWithEcConfig(t *testing.T) {
|
||||
dataDir, idxDir := t.TempDir(), t.TempDir()
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 11},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed local .vif: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(idxDir, "1.ecx"), []byte{}, 0644); err != nil {
|
||||
t.Fatalf("seed .ecx: %v", err)
|
||||
}
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 22},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed index .vif: %v", err)
|
||||
}
|
||||
|
||||
ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", needle.VolumeId(1))
|
||||
if err != nil {
|
||||
t.Fatalf("mount: %v", err)
|
||||
}
|
||||
defer ev.Close()
|
||||
if ev.EncodeTsNs != 11 {
|
||||
t.Errorf("EncodeTsNs = %d, want the local 11", ev.EncodeTsNs)
|
||||
}
|
||||
}
|
||||
|
||||
// With a shared -dir.idx the encode's .vif sits in the data dir of the disk
|
||||
// that received the index, so a disk holding a source volume's config-free
|
||||
// .vif takes it from that sibling, but only when the siblings agree on one run.
|
||||
func TestNewEcVolumeTakesSiblingVifWithSharedIndex(t *testing.T) {
|
||||
const runA, runB = int64(1_700_000_000_000_000_011), int64(1_700_000_000_000_000_022)
|
||||
cases := []struct {
|
||||
name string
|
||||
siblingTs []int64
|
||||
localIndex bool
|
||||
want int64
|
||||
}{
|
||||
{name: "one sibling", siblingTs: []int64{runA}, want: runA},
|
||||
{name: "siblings agree", siblingTs: []int64{runA, runA}, want: runA},
|
||||
{name: "siblings disagree", siblingTs: []int64{runA, runB}, want: 0},
|
||||
{name: "local index", siblingTs: []int64{runA}, localIndex: true, want: 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir, idxDir := t.TempDir(), t.TempDir()
|
||||
dirs := []string{dataDir}
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(dataDir, "c_1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
ReadOnly: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed source .vif: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(idxDir, "c_1.ecx"), make([]byte, 20), 0644); err != nil {
|
||||
t.Fatalf("seed .ecx: %v", err)
|
||||
}
|
||||
if tc.localIndex {
|
||||
if err := os.WriteFile(filepath.Join(dataDir, "c_1.ecx"), make([]byte, 20), 0644); err != nil {
|
||||
t.Fatalf("seed local .ecx: %v", err)
|
||||
}
|
||||
}
|
||||
for _, ts := range tc.siblingTs {
|
||||
sibling := t.TempDir()
|
||||
dirs = append(dirs, sibling)
|
||||
if err := volume_info.SaveVolumeInfo(filepath.Join(sibling, "c_1.vif"), &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(needle.Version3),
|
||||
DatFileSize: 4096,
|
||||
EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: ts},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed sibling .vif: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "c", needle.VolumeId(1), dirs...)
|
||||
if err != nil {
|
||||
t.Fatalf("mount: %v", err)
|
||||
}
|
||||
defer ev.Close()
|
||||
if ev.EncodeTsNs != tc.want {
|
||||
t.Errorf("EncodeTsNs = %d, want %d", ev.EncodeTsNs, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user