fix(ec): fence stale-worker EC shard cleanup by encode generation (#9953)

* feat(ec): add encode_ts_ns to the EC task params, shard-unmount, and shard-delete RPCs

The generation fence for stale EC-worker cleanup needs the encode
generation on three messages: ErasureCodingTaskParams (admin issues it),
VolumeEcShardsUnmountRequest, and VolumeEcShardsDeleteRequest (the worker
carries it to the volume server). Additive fields only; 0 preserves the
existing unfenced behavior. Mirror the two volume-server fields in the
Rust volume server's proto copy.

* feat(ec): issue the EC encode generation from the admin and carry it on the worker

Stamp each EC proposal's encode_ts_ns from the admin's per-cycle
DetectionSequence (a single-clock value) so generations are globally
ordered even though detection runs on a rotating worker. The worker
writes that generation into the distributed .vif and passes it on its
shard unmount/delete RPCs; it falls back to a local timestamp for the
.vif only on the unfenced legacy/shell path (keeping the read guard on).

* fix(ec): fence the stale-worker EC shard unmount and teardown by generation

A reaped-but-still-running EC worker's cleanupStaleEcShards issued a
generation-blind unmount + full teardown that could unmount and then
overwrite a newer run's live shards on a shared node. Both RPCs now
carry the encode generation: the volume server unmounts/deletes a disk
only when its .vif generation is strictly older than the request, and
preserves a same-or-newer generation, a generation-0 (recovered or
pre-upgrade) volume, and an unreadable .vif. Unload is per-disk, never
node-wide. Request generation 0 keeps the blanket teardown for the shell
pre-encode cleanup and pre-upgrade callers. Mirrored in the Rust volume
server.

* test(ec): cover the generation-fenced teardown and unmount

End-to-end volume-server tests: a fenced FullTeardown wipes a strictly-
older generation, preserves a newer one, preserves a generation-0 volume,
and blanket-wipes on request generation 0; the gen-aware unmount preserves
a same-or-newer mounted generation; and the .vif generation reader handles
present/absent/no-config cases.

* test(ec): pin the fenced .vif==teardown generation and the unreadable-.vif preserve

A fenced run must stamp the admin generation verbatim into the .vif so it
matches the generation sent on the teardown RPCs; add a regression test
that sets the task generation and asserts the .vif carries it exactly.
Also cover the present-but-unparseable .vif case (reads as generation 0,
preserved) and correct the readEcGenerationTsNs docstring accordingly.

* fix(ec): surface EC full-teardown filesystem errors in the Rust volume server

remove_ec_volume_files(_full_teardown) discarded every fs::remove_file
error, so a teardown that failed on permissions or a full disk still
returned full_teardown_done=true and left stale artifacts to collide with
the next encode. Return io::Result, ignore NotFound, propagate the first
real error, and have the teardown RPC surface it -- matching the Go
contract. The best-effort reconcile/load-cleanup callers keep ignoring it.

* refactor(ec): reuse the EC volume lookup on unmount and short-circuit the gen read

Address review: the Rust unmount fence reuses the ec_vol it already
fetched instead of a second find_ec_volume; the Go .vif generation reader
breaks out of the data/idx loop early when the two dirs are the same.

weed/server/volume_grpc_ec_generation_fence_test.go

@@ -0,0 +1,185 @@
+package weed_server
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/require"
+)
+
+// buildEcStoreWithGeneration creates a single-disk store holding one EC volume
+// whose .vif records the given encode generation, with the given shards mounted.
+func buildEcStoreWithGeneration(t *testing.T, dir, collection string, vid needle.VolumeId, encodeTsNs int64, shardIds []erasure_coding.ShardId) *storage.Store {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(dir, 0o755))
+	store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
+		storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-store.NewEcShardsChan:
+			case <-store.NewVolumesChan:
+			case <-store.DeletedVolumesChan:
+			case <-store.DeletedEcShardsChan:
+			case <-store.StateUpdateChan:
+			case <-done:
+				return
+			}
+		}
+	}()
+	t.Cleanup(func() {
+		store.Close()
+		close(done)
+	})
+
+	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
+	require.NoError(t, os.WriteFile(base+".ecx", make([]byte, 16), 0o644))
+	require.NoError(t, os.WriteFile(base+".ecj", nil, 0o644))
+	require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version:     uint32(needle.Version3),
+		DatFileSize: 10 * 1024 * 1024,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   10,
+			ParityShards: 4,
+			EncodeTsNs:   encodeTsNs,
+		},
+	}))
+	for _, sid := range shardIds {
+		f, err := os.Create(base + erasure_coding.ToExt(int(sid)))
+		require.NoError(t, err)
+		require.NoError(t, f.Truncate(1))
+		require.NoError(t, f.Close())
+	}
+	for _, sid := range shardIds {
+		require.NoError(t, store.MountEcShards(collection, vid, sid, ""))
+	}
+	return store
+}
+
+func mountedEcShardIds(t *testing.T, vs *VolumeServer, vid needle.VolumeId) map[int]bool {
+	t.Helper()
+	resp, err := vs.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: uint32(vid)})
+	require.NoError(t, err)
+	ids := make(map[int]bool)
+	for _, info := range resp.GetEcShardInfos() {
+		ids[int(info.GetShardId())] = true
+	}
+	return ids
+}
+
+// TestFullTeardownFencedByGeneration pins the finding-27 data-safety rule: a
+// generation-fenced FullTeardown deletes a disk whose .vif generation is strictly
+// older than the request, but preserves a same-or-newer generation, a generation-0
+// (recovered/pre-upgrade) volume, and falls back to a blanket wipe for request 0.
+func TestFullTeardownFencedByGeneration(t *testing.T) {
+	const collection = "ec-fence"
+	vid := needle.VolumeId(55)
+	shardIds := []erasure_coding.ShardId{0, 1}
+	shardExists := func(dir string) bool {
+		base := erasure_coding.EcShardFileName(collection, dir, int(vid))
+		return util.FileExists(base + erasure_coding.ToExt(0))
+	}
+	teardown := func(vs *VolumeServer, reqGen int64) {
+		_, err := vs.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
+			VolumeId:     uint32(vid),
+			Collection:   collection,
+			FullTeardown: true,
+			EncodeTsNs:   reqGen,
+		})
+		require.NoError(t, err)
+	}
+
+	t.Run("older_disk_wiped", func(t *testing.T) {
+		dir := t.TempDir()
+		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 100, shardIds)}
+		teardown(vs, 200) // request newer than the disk's generation 100
+		require.False(t, shardExists(dir), "a strictly-older generation must be wiped")
+	})
+
+	t.Run("newer_disk_preserved", func(t *testing.T) {
+		dir := t.TempDir()
+		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
+		teardown(vs, 100) // request older than the disk's generation 200
+		require.True(t, shardExists(dir), "a newer generation (a live newer run) must be preserved")
+	})
+
+	t.Run("zero_gen_preserved", func(t *testing.T) {
+		dir := t.TempDir()
+		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 0, shardIds)}
+		teardown(vs, 200) // a recovered/pre-upgrade live volume reports generation 0
+		require.True(t, shardExists(dir), "a generation-0 volume must be preserved under a fenced teardown")
+	})
+
+	t.Run("zero_request_blanket_wipe", func(t *testing.T) {
+		dir := t.TempDir()
+		vs := &VolumeServer{store: buildEcStoreWithGeneration(t, dir, collection, vid, 200, shardIds)}
+		teardown(vs, 0) // shell pre-encode / pre-upgrade caller wipes everything
+		require.False(t, shardExists(dir), "request generation 0 must blanket-wipe")
+	})
+}
+
+// TestUnmountEcShardsFencedByGeneration pins that the gen-aware unmount (issued
+// before the teardown) preserves a same-or-newer mounted generation, so a stale
+// worker cannot unmount a newer run's live shards out from under it.
+func TestUnmountEcShardsFencedByGeneration(t *testing.T) {
+	const collection = "ec-unmount-fence"
+	vid := needle.VolumeId(56)
+	dir := t.TempDir()
+	store := buildEcStoreWithGeneration(t, dir, collection, vid, 200, []erasure_coding.ShardId{0, 1})
+	vs := &VolumeServer{store: store}
+
+	// Request older than the disk generation 200: preserve (skip unmount).
+	require.NoError(t, store.UnmountEcShards(vid, 0, 100))
+	require.True(t, mountedEcShardIds(t, vs, vid)[0], "a same-or-newer generation shard must stay mounted")
+
+	// Request newer than the disk generation 200: a genuinely-older leftover is unmounted.
+	require.NoError(t, store.UnmountEcShards(vid, 1, 300))
+	require.False(t, mountedEcShardIds(t, vs, vid)[1], "a strictly-older generation shard must be unmounted")
+}
+
+// TestReadEcGenerationTsNs covers the per-disk .vif generation read used by the
+// fenced teardown: a present .vif yields its generation (or 0 when it has no EC
+// config), and a missing .vif is reported unreadable (preserved, fail-safe).
+func TestReadEcGenerationTsNs(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "9")
+
+	if _, readable := readEcGenerationTsNs(base, base); readable {
+		t.Fatalf("a missing .vif must be reported unreadable")
+	}
+
+	require.NoError(t, volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version:       uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4, EncodeTsNs: 12345},
+	}))
+	gen, readable := readEcGenerationTsNs(base, base)
+	require.True(t, readable)
+	require.Equal(t, int64(12345), gen)
+
+	// A .vif with no EC config (recovered / live source volume) reads as generation 0.
+	noCfg := filepath.Join(dir, "10")
+	require.NoError(t, volume_info.SaveVolumeInfo(noCfg+".vif", &volume_server_pb.VolumeInfo{Version: uint32(needle.Version3)}))
+	gen0, readable0 := readEcGenerationTsNs(noCfg, noCfg)
+	require.True(t, readable0)
+	require.Equal(t, int64(0), gen0)
+
+	// A present-but-unparseable .vif is reported as generation 0 (present), which the
+	// fenced teardown still preserves — never wiping on a parse error.
+	corrupt := filepath.Join(dir, "11")
+	require.NoError(t, os.WriteFile(corrupt+".vif", []byte("not-a-valid-vif"), 0o644))
+	genC, readableC := readEcGenerationTsNs(corrupt, corrupt)
+	require.True(t, readableC)
+	require.Equal(t, int64(0), genC)
+}

weed/server/volume_grpc_erasure_coding.go

@@ -418,17 +418,38 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
 	bName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId))
 
 	if req.FullTeardown {
-		// Pre-encode cleanup: evict the volume and wipe every EC artifact for it on
-		// every disk (the same teardown the generator does locally), not just the
-		// listed shards, so a remote node retains no stale generation that a fresh
-		// gen-0 copy would later collide with.
-		glog.V(0).Infof("ec volume %s full teardown", bName)
-		vs.store.UnloadEcVolume(needle.VolumeId(req.VolumeId))
+		if req.EncodeTsNs == 0 {
+			// Blanket teardown (shell pre-encode cleanup / pre-upgrade caller): evict the
+			// volume and wipe every EC artifact for it on every disk, not just the listed
+			// shards, so a remote node retains no stale generation a fresh copy collides with.
+			glog.V(0).Infof("ec volume %s full teardown", bName)
+			vs.store.UnloadEcVolume(needle.VolumeId(req.VolumeId))
+			for _, location := range vs.store.Locations {
+				dataBase := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
+				idxBase := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId))
+				if err := removeStaleEcArtifacts(dataBase, idxBase, erasure_coding.MaxShardCount); err != nil {
+					return nil, fmt.Errorf("full teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
+				}
+			}
+			return &volume_server_pb.VolumeEcShardsDeleteResponse{FullTeardownDone: true}, nil
+		}
+		// Generation-fenced teardown (stale-worker pre-distribute cleanup): wipe only a
+		// disk whose .vif generation is strictly OLDER than the request; preserve
+		// same-or-newer, generation 0 (recovered/pre-upgrade live volume), and an
+		// unreadable .vif, so a stale run can never wipe a newer run's live shards.
+		// Unload and remove only the strictly-older disks, never node-wide.
+		glog.V(0).Infof("ec volume %s full teardown fenced at generation %d", bName, req.EncodeTsNs)
 		for _, location := range vs.store.Locations {
 			dataBase := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
 			idxBase := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId))
+			diskGen, readable := readEcGenerationTsNs(dataBase, idxBase)
+			if !readable || diskGen == 0 || diskGen >= req.EncodeTsNs {
+				glog.V(1).Infof("ec volume %d on %s preserved: disk generation %d (readable=%v) not older than request %d", req.VolumeId, location.Directory, diskGen, readable, req.EncodeTsNs)
+				continue
+			}
+			location.UnloadEcVolume(needle.VolumeId(req.VolumeId))
 			if err := removeStaleEcArtifacts(dataBase, idxBase, erasure_coding.MaxShardCount); err != nil {
-				return nil, fmt.Errorf("full teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
+				return nil, fmt.Errorf("fenced teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
 			}
 		}
 		return &volume_server_pb.VolumeEcShardsDeleteResponse{FullTeardownDone: true}, nil
@@ -534,6 +555,24 @@ func removeFileIfExists(path string) error {
 	return nil
 }
 
+// readEcGenerationTsNs returns the EC encode generation recorded in a disk's .vif
+// (data dir first, then idx dir for the split-disk layout) and whether a .vif file
+// was present. A present .vif with no EcShardConfig — or one that failed to parse —
+// yields (0, true): generation 0, which the fenced teardown preserves anyway (a
+// recovered or pre-upgrade live volume). (0, false) means no .vif file was found.
+// Both 0 and a missing .vif are preserved, so the read is fail-safe in every case.
+func readEcGenerationTsNs(dataBaseFileName, indexBaseFileName string) (int64, bool) {
+	for _, base := range []string{dataBaseFileName, indexBaseFileName} {
+		if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(base + ".vif"); found {
+			return vi.GetEcShardConfig().GetEncodeTsNs(), true
+		}
+		if dataBaseFileName == indexBaseFileName {
+			break
+		}
+	}
+	return 0, false
+}
+
 // removeStaleEcArtifacts deletes the shard, index, journal, and bitrot sidecar
 // files of a prior encode so a fresh encode never mixes runs. total is the
 // shard-id range to scan (pass the cap for custom ratios). Returns the first
@@ -663,7 +702,7 @@ func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_s
 	glog.V(0).Infof("VolumeEcShardsUnmount: %v", req)
 
 	for _, shardId := range req.ShardIds {
-		err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId))
+		err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.EncodeTsNs)
 
 		if err != nil {
 			glog.Errorf("ec shard unmount %v: %v", req, err)