fix(ec): bring ec.encode worker and EC/volume helpers to parity with shell (#9599)

* refactor(volume): extract replica sync/select into shared volume_replica package Move the volume replica reconciliation helpers (status, union builder, SyncAndSelectBestReplica, ReadNeedleMeta) out of the shell into a new weed/storage/volume_replica package so both the shell (ec.encode, volume.tier.move, volume.check.disk) and the EC encode worker can reuse them. No behavior change. * fix(ec): bring ec.encode worker to parity with the shell - Sync replicas and encode the most-complete one (via the shared volume_replica.SyncAndSelectBestReplica) instead of a possibly-stale replica, marking all replicas readonly first. Prevents silent data loss when a stale replica is encoded and the originals deleted. - Skip remote/tiered volumes in detection (shell ec.encode excludes them). - Min-node safety gate: refuse to encode when cluster nodes < parity shards. - Align default thresholds with the shell (fullness 0.95, quiet 1h). * fix(vacuum): plugin path honors min_volume_age_seconds override deriveVacuumConfig hard-coded MinVolumeAgeSeconds=0, dropping any configured value. Read it from worker config (default 0, matching the shell/master vacuum which has no age gate) so an explicit override is honored. * address review feedback - config.go: align GetConfigSpec schema defaults (quiet_for_seconds=3600, fullness_ratio=0.95) with the runtime defaults so UI/bootstrap flows match the shell (coderabbitai). - ec_task.go: roll back readonly when markReplicasReadonly fails partway, so already-marked replicas don't stay readonly (coderabbitai). - volume_replica: pass the caller's replica statuses into buildUnionReplica instead of re-fetching them, and skip the per-needle ReadNeedleMeta RPC when the source replica is read-only (gemini-code-assist). * test(plugin_workers/ec): make fixtures eligible under the new defaults The default EC encode thresholds were raised to match the shell (fullness 0.95, quiet 1h), but the plugin-worker integration fixtures still used 90%-full / 10-minute-old volumes, so detection found no eligible volumes and the tests failed in CI. Bump the eligible fixtures to 96% full and 2h old.
2026-10-22 05:54:24 +02:00 · 2026-05-21 02:16:28 -07:00
1 parent 3f6410fdc3
commit cd15ae1395
@@ -1,6 +1,7 @@
 package erasure_coding

 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -19,6 +20,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
 	"google.golang.org/grpc"
@@ -35,12 +38,13 @@ type ErasureCodingTask struct {
 	grpcDialOption grpc.DialOption

 	// EC parameters
-	dataShards      int32
-	parityShards    int32
-	sourceDiskType  string                  // source volume's disk type, forwarded to Mount RPC (#9423)
-	targets         []*worker_pb.TaskTarget // Unified targets for EC shards
-	sources         []*worker_pb.TaskSource // Unified sources for cleanup
-	shardAssignment map[string][]string     // destination -> assigned shard types
+	dataShards       int32
+	parityShards     int32
+	sourceDiskType   string                  // source volume's disk type, forwarded to Mount RPC (#9423)
+	targets          []*worker_pb.TaskTarget // Unified targets for EC shards
+	sources          []*worker_pb.TaskSource // Unified sources for cleanup
+	shardAssignment  map[string][]string     // destination -> assigned shard types
+	readonlyReplicas []pb.ServerAddress      // replicas marked readonly, for rollback
 }

 // NewErasureCodingTask creates a new unified EC task instance
@@ -146,12 +150,22 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 		}
 	}()

-	// Step 1: Mark volume readonly
+	// Step 1: Mark all replicas readonly, then reconcile them and select the most
+	// complete replica as the encode source. Encoding a stale replica and then
+	// deleting the originals would silently lose entries that exist only on another
+	// replica; SyncAndSelectBestReplica builds the union onto the best replica first
+	// (mirrors the shell ec.encode best-replica selection).
 	t.ReportProgressWithStage(10.0, "Marking volume readonly")
 	t.GetLogger().Info("Marking volume readonly")
-	if err := t.markVolumeReadonly(ctx); err != nil {
+	if err := t.markReplicasReadonly(ctx); err != nil {
+		// Marking can fail partway; restore the replicas already marked readonly.
+		t.rollbackReadonly(ctx)
 		return fmt.Errorf("failed to mark volume readonly: %v", err)
 	}
+	if err := t.syncAndSelectSourceReplica(); err != nil {
+		t.rollbackReadonly(ctx)
+		return fmt.Errorf("failed to sync and select source replica: %v", err)
+	}

 	// Step 2: Copy volume files to worker
 	// The .idx and .dat are copied as separate network transfers, with .idx
@@ -277,39 +291,87 @@ func (t *ErasureCodingTask) GetProgress() float64 {

 // Helper methods for actual EC operations

-// markVolumeReadonly marks the volume as readonly on the source server
-func (t *ErasureCodingTask) markVolumeReadonly(ctx context.Context) error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
-		func(client volume_server_pb.VolumeServerClient) error {
-			_, err := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
-				VolumeId: t.volumeID,
-			})
-			return err
-		})
+// replicaLocations returns the regular (non-EC) volume replica locations from the
+// task sources. EC-shard sources carry shard ids; regular replicas do not. Falls
+// back to the assigned source server when no replica sources are present.
+func (t *ErasureCodingTask) replicaLocations() []wdclient.Location {
+	var locs []wdclient.Location
+	for _, s := range t.sources {
+		if s == nil || len(s.ShardIds) > 0 || s.Node == "" {
+			continue
+		}
+		locs = append(locs, wdclient.Location{Url: s.Node, DataCenter: s.DataCenter})
+	}
+	if len(locs) == 0 {
+		locs = append(locs, wdclient.Location{Url: t.server})
+	}
+	return locs
 }

-// rollbackReadonly is a best-effort rollback of markVolumeReadonly, used when the
-// EC task fails before any shards are distributed. Logs but does not return errors.
-// Uses a fresh context with timeout since the caller's ctx may already be cancelled.
+// markReplicasReadonly marks every regular replica readonly so no writes land
+// during encoding, recording them so rollbackReadonly can restore them all.
+func (t *ErasureCodingTask) markReplicasReadonly(ctx context.Context) error {
+	t.readonlyReplicas = t.readonlyReplicas[:0]
+	for _, loc := range t.replicaLocations() {
+		addr := loc.ServerAddress()
+		err := operation.WithVolumeServerClient(false, addr, t.grpcDialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				_, e := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: t.volumeID})
+				return e
+			})
+		if err != nil {
+			return fmt.Errorf("mark volume %d readonly on %s: %w", t.volumeID, addr, err)
+		}
+		t.readonlyReplicas = append(t.readonlyReplicas, addr)
+	}
+	return nil
+}
+
+// syncAndSelectSourceReplica reconciles the volume's replicas (building the union
+// of all live entries onto the most complete one) and switches the encode source
+// to that replica, so a stale replica is never the basis of the encode.
+func (t *ErasureCodingTask) syncAndSelectSourceReplica() error {
+	locs := t.replicaLocations()
+	if len(locs) <= 1 {
+		return nil // single replica: nothing to reconcile
+	}
+	var buf bytes.Buffer
+	best, err := volume_replica.SyncAndSelectBestReplica(t.grpcDialOption, needle.VolumeId(t.volumeID), t.collection, locs, "", &buf)
+	if out := strings.TrimSpace(buf.String()); out != "" {
+		glog.Infof("EC encode replica sync for volume %d:\n%s", t.volumeID, out)
+	}
+	if err != nil {
+		return err
+	}
+	if best.Url != "" && best.Url != t.server {
+		glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server)
+		t.server = best.Url
+	}
+	return nil
+}
+
+// rollbackReadonly is a best-effort restore of every replica markReplicasReadonly
+// touched, used when the EC task fails before the originals are deleted. Logs but
+// does not return errors; uses a fresh context since the caller's may be cancelled.
 func (t *ErasureCodingTask) rollbackReadonly(_ context.Context) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	if err := t.markVolumeWritable(ctx); err != nil {
-		glog.Warningf("failed to restore volume %d to writable after EC task failure: %v", t.volumeID, err)
-	} else {
-		glog.V(0).Infof("restored volume %d to writable after EC task failure", t.volumeID)
+	servers := t.readonlyReplicas
+	if len(servers) == 0 {
+		servers = []pb.ServerAddress{pb.ServerAddress(t.server)}
 	}
-}
-
-// markVolumeWritable restores the volume to writable on the source server.
-func (t *ErasureCodingTask) markVolumeWritable(ctx context.Context) error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
-		func(client volume_server_pb.VolumeServerClient) error {
-			_, err := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{
-				VolumeId: t.volumeID,
+	for _, addr := range servers {
+		err := operation.WithVolumeServerClient(false, addr, t.grpcDialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				_, e := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: t.volumeID})
+				return e
 			})
-			return err
-		})
+		if err != nil {
+			glog.Warningf("failed to restore volume %d to writable on %s after EC task failure: %v", t.volumeID, addr, err)
+		} else {
+			glog.V(0).Infof("restored volume %d to writable on %s after EC task failure", t.volumeID, addr)
+		}
+	}
 }

 // copyVolumeFilesToWorker copies .idx and .dat files from source server to local worker.