package weed_server
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/require"
)
// buildEcStoreWithGeneration creates a single-disk store holding one EC volume
// whose .vif records the given encode generation, with the given shards mounted.
func buildEcStoreWithGeneration(t *testing.T, dir, collection string, vid needle.VolumeId, encodeTsNs int64, shardIds []erasure_coding.ShardId) *storage.Store {
t.Helper()
require.NoError(t, os.MkdirAll(dir, 0o755))
store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
done := make(chan struct{})
go func() {
for {
select {
case <-store.NewEcShardsChan:
case <-store.NewVolumesChan:
case <-store.DeletedVolumesChan:
case <-store.DeletedEcShardsChan:
case <-store.StateUpdateChan:
case <-done:
return
}
}
}()
t.Cleanup(func() {
store.Close()
close(done)
})
base := erasure_coding.EcShardFileName(collection, dir, int(vid))
require.NoError(t, os.WriteFile(base+".ecx", make([]byte, 16), 0o644))
require.NoError(t, os.WriteFile(base+".ecj", nil, 0o644))
require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
Version: uint32(needle.Version3),
DatFileSize: 10 * 1024 * 1024,
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: 10,
ParityShards: 4,
EncodeTsNs: encodeTsNs,
},
}))
for _, sid := range shardIds {
f, err := os.Create(base + erasure_coding.ToExt(int(sid)))
require.NoError(t, err)
require.NoError(t, f.Truncate(1))
require.NoError(t, f.Close())
}
for _, sid := range shardIds {
require.NoError(t, store.MountEcShards(collection, vid, sid, ""))
}
return store
}
func mountedEcShardIds(t *testing.T, vs *VolumeServer, vid needle.VolumeId) map[int]bool {
t.Helper()
resp, err := vs.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: uint32(vid)})
require.NoError(t, err)
ids := make(map[int]bool)
for _, info := range resp.GetEcShardInfos() {
ids[int(info.GetShardId())] = true
}
return ids
}
// TestFullTeardownFencedByGeneration pins the finding-27 data-safety rule: a
// generation-fenced FullTeardown deletes a disk whose .vif generation is strictly
// older than the request, but preserves a same-or-newer generation, a generation-0
// (recovered/pre-upgrade) volume, and falls back to a blanket wipe for request 0.
func TestFullTeardownFencedByGeneration(t *testing.T) {
const collection = "ec-fence"
vid := needle.VolumeId(55)
shardIds := []erasure_coding.ShardId{0, 1}
shardExists := func(dir string) bool {
base := erasure_coding.EcShardFileName(collection, dir, int(vid))
return util.FileExists(base + erasure_coding.ToExt(0))
}
teardown := func(vs *VolumeServer, reqGen int64) {
_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(vid),
Collection: collection,
FullTeardown: true,
EncodeTsNs: reqGen,
})
require.NoError(t, err)
}
t.Run("older_disk_wiped", func(t *testing.T) {
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, shardIds)}
teardown(vs, 200) // request newer than the disk's generation 100
require.False(t, shardExists(dir), "a strictly-older generation must be wiped")
})
t.Run("newer_disk_preserved", func(t *testing.T) {
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
teardown(vs, 100) // request older than the disk's generation 200
require.True(t, shardExists(dir), "a newer generation (a live newer run) must be preserved")
})
t.Run("zero_gen_preserved", func(t *testing.T) {
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 0, shardIds)}
teardown(vs, 200) // a recovered/pre-upgrade live volume reports generation 0
require.True(t, shardExists(dir), "a generation-0 volume must be preserved under a fenced teardown")
})
t.Run("zero_request_blanket_wipe", func(t *testing.T) {
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
teardown(vs, 0) // shell pre-encode / pre-upgrade caller wipes everything
require.False(t, shardExists(dir), "request generation 0 must blanket-wipe")
})
}
// TestUnmountEcShardsFencedByGeneration pins that the gen-aware unmount (issued
// before the teardown) preserves a same-or-newer mounted generation, so a stale
// worker cannot unmount a newer run's live shards out from under it.
func TestUnmountEcShardsFencedByGeneration(t *testing.T) {
const collection = "ec-unmount-fence"
vid := needle.VolumeId(56)
dir := t.TempDir()
store := buildEcStoreWithGeneration(t, dir, collection, vid, 200, []erasure_coding.ShardId{0, 1})
vs := &VolumeServer{store: store}
// Request older than the disk generation 200: preserve (skip unmount).
require.NoError(t, store.UnmountEcShards(vid, 0, 100))
require.True(t, mountedEcShardIds(t, vs, vid)[0], "a same-or-newer generation shard must stay mounted")
// Request newer than the disk generation 200: a genuinely-older leftover is unmounted.
require.NoError(t, store.UnmountEcShards(vid, 1, 300))
require.False(t, mountedEcShardIds(t, vs, vid)[1], "a strictly-older generation shard must be unmounted")
}
// TestTeardownRemovesStagedGenerations pins that a full teardown wipes the
// 2PC-staged .*.v files together with the canonical ones: they are
// invisible to EC bookkeeping, so anything left behind leaks forever.
func TestTeardownRemovesStagedGenerations(t *testing.T) {
const collection = "ec-gen-leak"
vid := needle.VolumeId(57)
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 1})}
base := erasure_coding.EcShardFileName(collection, dir, int(vid))
for _, name := range []string{
base + ".ec00.v3", base + ".ec01.v3", base + ".ecx.v3", base + ".vif.v3", base + ".ecsum.v3",
} {
require.NoError(t, os.WriteFile(name, []byte("staged"), 0o644))
}
_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(vid),
Collection: collection,
FullTeardown: true,
EncodeTsNs: 200,
})
require.NoError(t, err)
left, err := filepath.Glob(base + "*")
require.NoError(t, err)
require.Empty(t, left, "teardown must leave no EC files, staged generations included: %v", left)
}
// TestDeleteGenerationsOlderThan covers the post-commit cleanup: staged
// generations below the threshold are removed while the committed generation
// and the live canonical files stay untouched.
func TestDeleteGenerationsOlderThan(t *testing.T) {
const collection = "ec-gen-gc"
vid := needle.VolumeId(58)
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 1})}
base := erasure_coding.EcShardFileName(collection, dir, int(vid))
stale := []string{base + ".ec00.v3", base + ".ecx.v3", base + ".vif.v3"}
fresh := []string{base + ".ec00.v7", base + ".vif.v7"}
for _, name := range append(stale, fresh...) {
require.NoError(t, os.WriteFile(name, []byte("staged"), 0o644))
}
_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(vid),
Collection: collection,
DeleteGenerationsOlderThan: 5,
})
require.NoError(t, err)
for _, name := range stale {
require.False(t, util.FileExists(name), "%s must be removed", name)
}
for _, name := range fresh {
require.True(t, util.FileExists(name), "%s must be preserved", name)
}
require.True(t, util.FileExists(base+".ec00"), "canonical shards must be preserved")
require.True(t, util.FileExists(base+".ecx"))
require.True(t, util.FileExists(base+".vif"))
require.True(t, mountedEcShardIds(t, vs, vid)[0], "mounted shards must stay mounted")
}
// TestShardDeleteRemovesStagedGenerations pins that deleting a shard removes
// its staged generations too: a shard evicted off a disk leaves nothing.
func TestShardDeleteRemovesStagedGenerations(t *testing.T) {
const collection = "ec-shard-gen"
vid := needle.VolumeId(59)
dir := t.TempDir()
vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 5})}
base := erasure_coding.EcShardFileName(collection, dir, int(vid))
require.NoError(t, os.WriteFile(base+".ec05.v2", []byte("staged"), 0o644))
require.NoError(t, vs.store.UnmountEcShards(vid, 5, 0))
_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(vid),
Collection: collection,
ShardIds: []uint32{5},
})
require.NoError(t, err)
require.False(t, util.FileExists(base+".ec05"))
require.False(t, util.FileExists(base+".ec05.v2"))
require.True(t, util.FileExists(base+".ec00"), "sibling shards must be preserved")
require.True(t, util.FileExists(base+".ecx"), "index must survive while shards remain")
}
// TestReadEcGenerationTsNs covers the per-disk .vif generation read used by the
// fenced teardown: a present .vif yields its generation (or 0 when it has no EC
// config), and a missing .vif is reported unreadable (preserved, fail-safe).
func TestReadEcGenerationTsNs(t *testing.T) {
dir := t.TempDir()
base := filepath.Join(dir, "9")
if _, readable := readEcGenerationTsNs(base, base); readable {
t.Fatalf("a missing .vif must be reported unreadable")
}
require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
Version: uint32(needle.Version3),
EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 12345},
}))
gen, readable := readEcGenerationTsNs(base, base)
require.True(t, readable)
require.Equal(t, int64(12345), gen)
// A .vif with no EC config (recovered / live source volume) reads as generation 0.
noCfg := filepath.Join(dir, "10")
require.NoError(t, volume_info.SaveVolumeInfo(noCfg+".vif", &volume_server_pb.VolumeInfo{Version: uint32(needle.Version3)}))
gen0, readable0 := readEcGenerationTsNs(noCfg, noCfg)
require.True(t, readable0)
require.Equal(t, int64(0), gen0)
// A present-but-unparseable .vif is reported as generation 0 (present), which the
// fenced teardown still preserves — never wiping on a parse error.
corrupt := filepath.Join(dir, "11")
require.NoError(t, os.WriteFile(corrupt+".vif", []byte("not-a-valid-vif"), 0o644))
genC, readableC := readEcGenerationTsNs(corrupt, corrupt)
require.True(t, readableC)
require.Equal(t, int64(0), genC)
}