fix(ec): fence stale-worker EC shard cleanup by encode generation (#9953)

* feat(ec): add encode_ts_ns to the EC task params, shard-unmount, and shard-delete RPCs

The generation fence for stale EC-worker cleanup needs the encode
generation on three messages: ErasureCodingTaskParams (admin issues it),
VolumeEcShardsUnmountRequest, and VolumeEcShardsDeleteRequest (the worker
carries it to the volume server). Additive fields only; 0 preserves the
existing unfenced behavior. Mirror the two volume-server fields in the
Rust volume server's proto copy.

* feat(ec): issue the EC encode generation from the admin and carry it on the worker

Stamp each EC proposal's encode_ts_ns from the admin's per-cycle
DetectionSequence (a single-clock value) so generations are globally
ordered even though detection runs on a rotating worker. The worker
writes that generation into the distributed .vif and passes it on its
shard unmount/delete RPCs; it falls back to a local timestamp for the
.vif only on the unfenced legacy/shell path (keeping the read guard on).

* fix(ec): fence the stale-worker EC shard unmount and teardown by generation

A reaped-but-still-running EC worker's cleanupStaleEcShards issued a
generation-blind unmount + full teardown that could unmount and then
overwrite a newer run's live shards on a shared node. Both RPCs now
carry the encode generation: the volume server unmounts/deletes a disk
only when its .vif generation is strictly older than the request, and
preserves a same-or-newer generation, a generation-0 (recovered or
pre-upgrade) volume, and an unreadable .vif. Unload is per-disk, never
node-wide. Request generation 0 keeps the blanket teardown for the shell
pre-encode cleanup and pre-upgrade callers. Mirrored in the Rust volume
server.

* test(ec): cover the generation-fenced teardown and unmount

End-to-end volume-server tests: a fenced FullTeardown wipes a strictly-
older generation, preserves a newer one, preserves a generation-0 volume,
and blanket-wipes on request generation 0; the gen-aware unmount preserves
a same-or-newer mounted generation; and the .vif generation reader handles
present/absent/no-config cases.

* test(ec): pin the fenced .vif==teardown generation and the unreadable-.vif preserve

A fenced run must stamp the admin generation verbatim into the .vif so it
matches the generation sent on the teardown RPCs; add a regression test
that sets the task generation and asserts the .vif carries it exactly.
Also cover the present-but-unparseable .vif case (reads as generation 0,
preserved) and correct the readEcGenerationTsNs docstring accordingly.

* fix(ec): surface EC full-teardown filesystem errors in the Rust volume server

remove_ec_volume_files(_full_teardown) discarded every fs::remove_file
error, so a teardown that failed on permissions or a full disk still
returned full_teardown_done=true and left stale artifacts to collide with
the next encode. Return io::Result, ignore NotFound, propagate the first
real error, and have the teardown RPC surface it -- matching the Go
contract. The best-effort reconcile/load-cleanup callers keep ignoring it.

* refactor(ec): reuse the EC volume lookup on unmount and short-circuit the gen read

Address review: the Rust unmount fence reuses the ec_vol it already
fetched instead of a second find_ec_volume; the Go .vif generation reader
breaks out of the data/idx loop early when the two dirs are the same.

weed/server/volume_grpc_erasure_coding.go

@@ -418,17 +418,38 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
 	bName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId))
 
 	if req.FullTeardown {
-		// Pre-encode cleanup: evict the volume and wipe every EC artifact for it on
-		// every disk (the same teardown the generator does locally), not just the
-		// listed shards, so a remote node retains no stale generation that a fresh
-		// gen-0 copy would later collide with.
-		glog.V(0).Infof("ec volume %s full teardown", bName)
-		vs.store.UnloadEcVolume(needle.VolumeId(req.VolumeId))
+		if req.EncodeTsNs == 0 {
+			// Blanket teardown (shell pre-encode cleanup / pre-upgrade caller): evict the
+			// volume and wipe every EC artifact for it on every disk, not just the listed
+			// shards, so a remote node retains no stale generation a fresh copy collides with.
+			glog.V(0).Infof("ec volume %s full teardown", bName)
+			vs.store.UnloadEcVolume(needle.VolumeId(req.VolumeId))
+			for _, location := range vs.store.Locations {
+				dataBase := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
+				idxBase := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId))
+				if err := removeStaleEcArtifacts(dataBase, idxBase, erasure_coding.MaxShardCount); err != nil {
+					return nil, fmt.Errorf("full teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
+				}
+			}
+			return &volume_server_pb.VolumeEcShardsDeleteResponse{FullTeardownDone: true}, nil
+		}
+		// Generation-fenced teardown (stale-worker pre-distribute cleanup): wipe only a
+		// disk whose .vif generation is strictly OLDER than the request; preserve
+		// same-or-newer, generation 0 (recovered/pre-upgrade live volume), and an
+		// unreadable .vif, so a stale run can never wipe a newer run's live shards.
+		// Unload and remove only the strictly-older disks, never node-wide.
+		glog.V(0).Infof("ec volume %s full teardown fenced at generation %d", bName, req.EncodeTsNs)
 		for _, location := range vs.store.Locations {
 			dataBase := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
 			idxBase := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId))
+			diskGen, readable := readEcGenerationTsNs(dataBase, idxBase)
+			if !readable || diskGen == 0 || diskGen >= req.EncodeTsNs {
+				glog.V(1).Infof("ec volume %d on %s preserved: disk generation %d (readable=%v) not older than request %d", req.VolumeId, location.Directory, diskGen, readable, req.EncodeTsNs)
+				continue
+			}
+			location.UnloadEcVolume(needle.VolumeId(req.VolumeId))
 			if err := removeStaleEcArtifacts(dataBase, idxBase, erasure_coding.MaxShardCount); err != nil {
-				return nil, fmt.Errorf("full teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
+				return nil, fmt.Errorf("fenced teardown of ec volume %d on %s: %w", req.VolumeId, location.Directory, err)
 			}
 		}
 		return &volume_server_pb.VolumeEcShardsDeleteResponse{FullTeardownDone: true}, nil
@@ -534,6 +555,24 @@ func removeFileIfExists(path string) error {
 	return nil
 }
 
+// readEcGenerationTsNs returns the EC encode generation recorded in a disk's .vif
+// (data dir first, then idx dir for the split-disk layout) and whether a .vif file
+// was present. A present .vif with no EcShardConfig — or one that failed to parse —
+// yields (0, true): generation 0, which the fenced teardown preserves anyway (a
+// recovered or pre-upgrade live volume). (0, false) means no .vif file was found.
+// Both 0 and a missing .vif are preserved, so the read is fail-safe in every case.
+func readEcGenerationTsNs(dataBaseFileName, indexBaseFileName string) (int64, bool) {
+	for _, base := range []string{dataBaseFileName, indexBaseFileName} {
+		if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(base + ".vif"); found {
+			return vi.GetEcShardConfig().GetEncodeTsNs(), true
+		}
+		if dataBaseFileName == indexBaseFileName {
+			break
+		}
+	}
+	return 0, false
+}
+
 // removeStaleEcArtifacts deletes the shard, index, journal, and bitrot sidecar
 // files of a prior encode so a fresh encode never mixes runs. total is the
 // shard-id range to scan (pass the cap for custom ratios). Returns the first
@@ -663,7 +702,7 @@ func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_s
 	glog.V(0).Infof("VolumeEcShardsUnmount: %v", req)
 
 	for _, shardId := range req.ShardIds {
-		err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId))
+		err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.EncodeTsNs)
 
 		if err != nil {
 			glog.Errorf("ec shard unmount %v: %v", req, err)