fix(ec): delete empty stub replicas before distributing EC shards (#9722)

* fix(ec): delete empty stub replicas before distributing EC shards

An interrupted encode can leave a 0-byte .dat replica behind. Until now
the only thing that removed it was deleteOriginalVolume, which runs after
distribute+mount and calls VolumeDelete -> removeVolumeFiles. A regular
volume and an EC volume share the same <collection>_<vid>.vif path, so
deleting the stub at that point strips the .vif out from under the
freshly distributed shards.

Sweep the original replicas with VolumeDelete(OnlyEmpty=true) before
distribute: doIsEmpty uses the same superblock threshold, so only the
0-byte stubs go and any data-bearing replica is refused and kept for the
post-verify delete. Servers cleared in the sweep are skipped by
deleteOriginalVolume so it never touches a server that now holds only EC
shards.

* fix(ec): fail the encode when an empty-replica sweep can't confirm a node

The sweep swallowed every VolumeDelete(OnlyEmpty) error, so a transient
failure on a stub node fell through to the post-verify force-delete on
that node — the shared-.vif clobber the sweep exists to avoid.

Treat only the expected cases (volume not empty, or already gone) as
leave-in-place; any other error propagates and fails the encode, which
rolls back the readonly marks and retries next cycle.

weed/worker/tasks/erasure_coding/ec_task.go

@@ -45,6 +45,11 @@ type ErasureCodingTask struct {
 	sources          []*worker_pb.TaskSource // Unified sources for cleanup
 	shardAssignment  map[string][]string     // destination -> assigned shard types
 	readonlyReplicas []pb.ServerAddress      // replicas marked readonly, for rollback
+
+	// Replica servers whose original volume was an empty stub, deleted in the
+	// pre-distribute sweep. deleteOriginalVolume skips these so it does not
+	// re-delete and remove the now-EC .vif those servers share.
+	emptyReplicasDeleted map[string]bool
 }
 
 // NewErasureCodingTask creates a new unified EC task instance
@@ -200,6 +205,18 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 		return fmt.Errorf("failed to clear stale EC shards on destinations: %v", err)
 	}
 
+	// Delete 0-byte stub replicas left by an interrupted encode before the new
+	// EC files land. A stub shares the <collection>_<vid>.vif path the EC
+	// volume will use; deleting it after distribute (in deleteOriginalVolume)
+	// would remove that .vif and damage the freshly written shards. OnlyEmpty
+	// keeps data-bearing replicas, which are deleted later after verify.
+	t.ReportProgressWithStage(57.0, "Removing empty stub replicas")
+	t.GetLogger().Info("Removing empty stub replicas before distribute")
+	if err := t.sweepEmptyReplicas(ctx); err != nil {
+		t.rollbackReadonly(ctx)
+		return fmt.Errorf("failed to remove empty stub replicas: %w", err)
+	}
+
 	// Step 4: Distribute shards to destinations
 	t.ReportProgressWithStage(60.0, "Distributing EC shards to destinations")
 	t.GetLogger().Info("Distributing EC shards to destinations")
@@ -675,6 +692,14 @@ func (t *ErasureCodingTask) deleteOriginalVolume(ctx context.Context) error {
 		replicas = []string{t.server}
 	}
 
+	// Empty stub replicas were already removed before distribute; skip them so
+	// VolumeDelete does not run on a server that now holds only EC shards.
+	replicas = replicasPendingDelete(replicas, t.emptyReplicasDeleted)
+	if len(replicas) == 0 {
+		glog.V(0).Infof("EC volume %d: all original replicas were empty stubs removed before distribute", t.volumeID)
+		return nil
+	}
+
 	t.GetLogger().WithFields(map[string]interface{}{
 		"volume_id":       t.volumeID,
 		"replica_count":   len(replicas),
@@ -762,6 +787,67 @@ func (t *ErasureCodingTask) getReplicas() []string {
 	return replicas
 }
 
+// sweepEmptyReplicas deletes any original replica that is an empty 0-byte stub
+// (OnlyEmpty so a data-bearing replica is refused and kept for the post-verify
+// delete). Run before distribute: a stub shares the <collection>_<vid>.vif the
+// EC volume reuses, so removing it afterwards would strip that .vif. Servers
+// whose stub was deleted are recorded so deleteOriginalVolume skips them.
+//
+// A refusal (volume not empty) or an already-gone volume is expected and left
+// for the later delete. Any other error means the node's state is unknown; we
+// fail rather than proceed to distribute and a force-delete that could strip a
+// shared .vif.
+func (t *ErasureCodingTask) sweepEmptyReplicas(ctx context.Context) error {
+	for _, node := range t.getReplicas() {
+		err := operation.WithVolumeServerClient(false, pb.ServerAddress(node), t.grpcDialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				_, e := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+					VolumeId:  t.volumeID,
+					OnlyEmpty: true,
+				})
+				return e
+			})
+		switch {
+		case err == nil:
+			if t.emptyReplicasDeleted == nil {
+				t.emptyReplicasDeleted = make(map[string]bool)
+			}
+			t.emptyReplicasDeleted[node] = true
+			glog.V(0).Infof("EC volume %d: removed empty stub replica on %s before distribute", t.volumeID, node)
+		case isExpectedSweepSkip(err):
+			glog.V(1).Infof("EC volume %d: empty-replica sweep left %s in place: %v", t.volumeID, node, err)
+		default:
+			return fmt.Errorf("empty-replica sweep on %s: %w", node, err)
+		}
+	}
+	return nil
+}
+
+// isExpectedSweepSkip reports whether a VolumeDelete(OnlyEmpty) error is the
+// expected leave-in-place case: the replica still holds data (refused) or no
+// longer exists. Other errors (e.g. an unreachable node) leave its state
+// unknown and must not be swallowed.
+func isExpectedSweepSkip(err error) bool {
+	s := err.Error()
+	return strings.Contains(s, "volume not empty") || strings.Contains(s, "not found")
+}
+
+// replicasPendingDelete returns replicas not already removed by the
+// pre-distribute empty-stub sweep.
+func replicasPendingDelete(replicas []string, alreadyDeleted map[string]bool) []string {
+	if len(alreadyDeleted) == 0 {
+		return replicas
+	}
+	pending := make([]string, 0, len(replicas))
+	for _, r := range replicas {
+		if alreadyDeleted[r] {
+			continue
+		}
+		pending = append(pending, r)
+	}
+	return pending
+}
+
 // cleanupStaleEcShards unmounts and deletes any EC shards still mounted on
 // destinations from a previous failed encode of this volume. Targets every
 // node we plan to write to (t.targets) plus every node detection saw EC