seaweedfs/weed/server/volume_grpc_ec_generation_fence_test.go

package weed_server

import (
	"context"
	"os"
	"path/filepath"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/stretchr/testify/require"
)

// buildEcStoreWithGeneration creates a single-disk store holding one EC volume
// whose .vif records the given encode generation, with the given shards mounted.
func buildEcStoreWithGeneration(t *testing.T, dir, collection string, vid needle.VolumeId, encodeTsNs int64, shardIds []erasure_coding.ShardId) *storage.Store {
	t.Helper()
	require.NoError(t, os.MkdirAll(dir, 0o755))
	store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
		storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
	done := make(chan struct{})
	go func() {
		for {
			select {
			case <-store.NewEcShardsChan:
			case <-store.NewVolumesChan:
			case <-store.DeletedVolumesChan:
			case <-store.DeletedEcShardsChan:
			case <-store.StateUpdateChan:
			case <-done:
				return
			}
		}
	}()
	t.Cleanup(func() {
		store.Close()
		close(done)
	})

	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
	require.NoError(t, os.WriteFile(base+".ecx", make([]byte, 16), 0o644))
	require.NoError(t, os.WriteFile(base+".ecj", nil, 0o644))
	require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
		Version:     uint32(needle.Version3),
		DatFileSize: 10 * 1024 * 1024,
		EcShardConfig: &volume_server_pb.EcShardConfig{
			DataShards:   10,
			ParityShards: 4,
			EncodeTsNs:   encodeTsNs,
		},
	}))
	for _, sid := range shardIds {
		f, err := os.Create(base + erasure_coding.ToExt(int(sid)))
		require.NoError(t, err)
		require.NoError(t, f.Truncate(1))
		require.NoError(t, f.Close())
	}
	for _, sid := range shardIds {
		require.NoError(t, store.MountEcShards(collection, vid, sid, ""))
	}
	return store
}

func mountedEcShardIds(t *testing.T, vs *VolumeServer, vid needle.VolumeId) map[int]bool {
	t.Helper()
	resp, err := vs.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: uint32(vid)})
	require.NoError(t, err)
	ids := make(map[int]bool)
	for _, info := range resp.GetEcShardInfos() {
		ids[int(info.GetShardId())] = true
	}
	return ids
}

// TestFullTeardownFencedByGeneration pins the finding-27 data-safety rule: a
// generation-fenced FullTeardown deletes a disk whose .vif generation is strictly
// older than the request, but preserves a same-or-newer generation, a generation-0
// (recovered/pre-upgrade) volume, and falls back to a blanket wipe for request 0.
func TestFullTeardownFencedByGeneration(t *testing.T) {
	const collection = "ec-fence"
	vid := needle.VolumeId(55)
	shardIds := []erasure_coding.ShardId{0, 1}
	shardExists := func(dir string) bool {
		base := erasure_coding.EcShardFileName(collection, dir, int(vid))
		return util.FileExists(base + erasure_coding.ToExt(0))
	}
	teardown := func(vs *VolumeServer, reqGen int64) {
		_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
			VolumeId:     uint32(vid),
			Collection:   collection,
			FullTeardown: true,
			EncodeTsNs:   reqGen,
		})
		require.NoError(t, err)
	}

	t.Run("older_disk_wiped", func(t *testing.T) {
		dir := t.TempDir()
		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, shardIds)}
		teardown(vs, 200) // request newer than the disk's generation 100
		require.False(t, shardExists(dir), "a strictly-older generation must be wiped")
	})

	t.Run("newer_disk_preserved", func(t *testing.T) {
		dir := t.TempDir()
		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
		teardown(vs, 100) // request older than the disk's generation 200
		require.True(t, shardExists(dir), "a newer generation (a live newer run) must be preserved")
	})

	t.Run("zero_gen_preserved", func(t *testing.T) {
		dir := t.TempDir()
		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 0, shardIds)}
		teardown(vs, 200) // a recovered/pre-upgrade live volume reports generation 0
		require.True(t, shardExists(dir), "a generation-0 volume must be preserved under a fenced teardown")
	})

	t.Run("zero_request_blanket_wipe", func(t *testing.T) {
		dir := t.TempDir()
		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
		teardown(vs, 0) // shell pre-encode / pre-upgrade caller wipes everything
		require.False(t, shardExists(dir), "request generation 0 must blanket-wipe")
	})
}

// TestUnmountEcShardsFencedByGeneration pins that the gen-aware unmount (issued
// before the teardown) preserves a same-or-newer mounted generation, so a stale
// worker cannot unmount a newer run's live shards out from under it.
func TestUnmountEcShardsFencedByGeneration(t *testing.T) {
	const collection = "ec-unmount-fence"
	vid := needle.VolumeId(56)
	dir := t.TempDir()
	store := buildEcStoreWithGeneration(t, dir, collection, vid, 200, []erasure_coding.ShardId{0, 1})
	vs := &VolumeServer{store: store}

	// Request older than the disk generation 200: preserve (skip unmount).
	require.NoError(t, store.UnmountEcShards(vid, 0, 100))
	require.True(t, mountedEcShardIds(t, vs, vid)[0], "a same-or-newer generation shard must stay mounted")

	// Request newer than the disk generation 200: a genuinely-older leftover is unmounted.
	require.NoError(t, store.UnmountEcShards(vid, 1, 300))
	require.False(t, mountedEcShardIds(t, vs, vid)[1], "a strictly-older generation shard must be unmounted")
}

// TestTeardownRemovesStagedGenerations pins that a full teardown wipes the
// 2PC-staged <base>.*.v<N> files together with the canonical ones: they are
// invisible to EC bookkeeping, so anything left behind leaks forever.
func TestTeardownRemovesStagedGenerations(t *testing.T) {
	const collection = "ec-gen-leak"
	vid := needle.VolumeId(57)
	dir := t.TempDir()
	vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 1})}

	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
	for _, name := range []string{
		base + ".ec00.v3", base + ".ec01.v3", base + ".ecx.v3", base + ".vif.v3", base + ".ecsum.v3",
	} {
		require.NoError(t, os.WriteFile(name, []byte("staged"), 0o644))
	}

	_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
		VolumeId:     uint32(vid),
		Collection:   collection,
		FullTeardown: true,
		EncodeTsNs:   200,
	})
	require.NoError(t, err)
	left, err := filepath.Glob(base + "*")
	require.NoError(t, err)
	require.Empty(t, left, "teardown must leave no EC files, staged generations included: %v", left)
}

// TestDeleteGenerationsOlderThan covers the post-commit cleanup: staged
// generations below the threshold are removed while the committed generation
// and the live canonical files stay untouched.
func TestDeleteGenerationsOlderThan(t *testing.T) {
	const collection = "ec-gen-gc"
	vid := needle.VolumeId(58)
	dir := t.TempDir()
	vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 1})}

	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
	stale := []string{base + ".ec00.v3", base + ".ecx.v3", base + ".vif.v3"}
	fresh := []string{base + ".ec00.v7", base + ".vif.v7"}
	for _, name := range append(stale, fresh...) {
		require.NoError(t, os.WriteFile(name, []byte("staged"), 0o644))
	}

	_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
		VolumeId:                   uint32(vid),
		Collection:                 collection,
		DeleteGenerationsOlderThan: 5,
	})
	require.NoError(t, err)
	for _, name := range stale {
		require.False(t, util.FileExists(name), "%s must be removed", name)
	}
	for _, name := range fresh {
		require.True(t, util.FileExists(name), "%s must be preserved", name)
	}
	require.True(t, util.FileExists(base+".ec00"), "canonical shards must be preserved")
	require.True(t, util.FileExists(base+".ecx"))
	require.True(t, util.FileExists(base+".vif"))
	require.True(t, mountedEcShardIds(t, vs, vid)[0], "mounted shards must stay mounted")
}

// TestShardDeleteRemovesStagedGenerations pins that deleting a shard removes
// its staged generations too: a shard evicted off a disk leaves nothing.
func TestShardDeleteRemovesStagedGenerations(t *testing.T) {
	const collection = "ec-shard-gen"
	vid := needle.VolumeId(59)
	dir := t.TempDir()
	vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, []erasure_coding.ShardId{0, 5})}

	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
	require.NoError(t, os.WriteFile(base+".ec05.v2", []byte("staged"), 0o644))
	require.NoError(t, vs.store.UnmountEcShards(vid, 5, 0))

	_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
		VolumeId:   uint32(vid),
		Collection: collection,
		ShardIds:   []uint32{5},
	})
	require.NoError(t, err)
	require.False(t, util.FileExists(base+".ec05"))
	require.False(t, util.FileExists(base+".ec05.v2"))
	require.True(t, util.FileExists(base+".ec00"), "sibling shards must be preserved")
	require.True(t, util.FileExists(base+".ecx"), "index must survive while shards remain")
}

// TestReadEcGenerationTsNs covers the per-disk .vif generation read used by the
// fenced teardown: a present .vif yields its generation (or 0 when it has no EC
// config), and a missing .vif is reported unreadable (preserved, fail-safe).
func TestReadEcGenerationTsNs(t *testing.T) {
	dir := t.TempDir()
	base := filepath.Join(dir, "9")

	if _, readable := readEcGenerationTsNs(base, base); readable {
		t.Fatalf("a missing .vif must be reported unreadable")
	}

	require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
		Version:       uint32(needle.Version3),
		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 12345},
	}))
	gen, readable := readEcGenerationTsNs(base, base)
	require.True(t, readable)
	require.Equal(t, int64(12345), gen)

	// A .vif with no EC config (recovered / live source volume) reads as generation 0.
	noCfg := filepath.Join(dir, "10")
	require.NoError(t, volume_info.SaveVolumeInfo(noCfg+".vif", &volume_server_pb.VolumeInfo{Version: uint32(needle.Version3)}))
	gen0, readable0 := readEcGenerationTsNs(noCfg, noCfg)
	require.True(t, readable0)
	require.Equal(t, int64(0), gen0)

	// A present-but-unparseable .vif is reported as generation 0 (present), which the
	// fenced teardown still preserves — never wiping on a parse error.
	corrupt := filepath.Join(dir, "11")
	require.NoError(t, os.WriteFile(corrupt+".vif", []byte("not-a-valid-vif"), 0o644))
	genC, readableC := readEcGenerationTsNs(corrupt, corrupt)
	require.True(t, readableC)
	require.Equal(t, int64(0), genC)
}