ec: compare live idx NeedleId sets before encode (#11812)

* ec: compare live idx NeedleId sets before encode

Refuse encode when replicas' live .idx NeedleId sets differ. Walk only
live rows (skip deleted and Size<=0) and compare NeedleId, ignoring Size
so MIME/flags drift does not abort a matching id set. Mark the volume
read-only for encode and reject writes after that.

* fix(ec): compare all replicas then pick capacity source

Log matching live idx sets with "replicas are consistent" for the
shell CI check. After readonly, compare every replica's live NeedleId
set, then require the encode source to have FreeVolumeCount>=2.
Empty live-idx vs non-empty refuses encode instead of skipping
superblock stubs. Drop the FileCount rebind after the idx gate.

* volume: mark readonly under the write lock so checked appends finish first

* ec: fix readonly deadlock and restore worker-host source preference

MarkVolumeReadonly held dataFileAccessLock while calling
reopenIdxForWrite, which re-acquires the same mutex and deadlocks
TestMarkVolumeReadonlyCanDelete. Split out reopenIdxForWriteLocked for
callers already holding the lock, matching the existing Locked pattern.

The worker task lost its worker-host source preference when the idx gate
replaced the replica union. Late-bind the source to a replica on the
worker host using only the checked location set, with no FileCount
rebinding. Drop the now-unused helpers, trim comments added by the
earlier commits, and rename tests whose names still claimed convergence
while asserting refusal.

---------

Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com>
Co-authored-by: Devin <devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/worker/tasks/erasure_coding/ec_task.go

@@ -181,11 +181,9 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 		return fmt.Errorf("EC preflight failed for volume %d: %w", t.volumeID, err)
 	}
 
-	// Step 1: Mark all replicas readonly, then reconcile them and select the most
-	// complete replica as the encode source. Encoding a stale replica and then
-	// deleting the originals would silently lose entries that exist only on another
-	// replica; SyncAndSelectBestReplica builds the union onto the best replica first
-	// (mirrors the shell ec.encode best-replica selection).
+	// Step 1: Mark all replicas readonly, then compare live .idx NeedleId
+	// sets. A mismatch or idx read error aborts encode and does not build a union.
+	// The encode source is late-bound only when the sets match.
 	t.ReportProgressWithStage(10.0, "Marking volume readonly")
 	t.GetLogger().Info("Marking volume readonly")
 	if err := t.markReplicasReadonly(ctx); err != nil {
@@ -391,90 +389,43 @@ func (t *ErasureCodingTask) markReplicasReadonly(ctx context.Context) error {
 	return nil
 }
 
-// syncAndSelectSourceReplica reconciles the volume's replicas (building the union
-// of all live entries onto the most complete one) and switches the encode source
-// to that replica, so a stale replica is never the basis of the encode.
+// syncAndSelectSourceReplica compares live .idx sets after every replica is
+// readonly. Matching sets late-bind the encode source; a mismatch or idx RPC
+// error aborts without unioning replicas.
 func (t *ErasureCodingTask) syncAndSelectSourceReplica() error {
 	locs := t.replicaLocations()
-	if len(locs) <= 1 {
-		return nil // single replica: nothing to reconcile
+	workerAddr := strings.TrimSpace(t.workerAddress)
+	if workerAddr == "" {
+		workerAddr, _ = os.Hostname()
 	}
+	prefer := lateBindSourceToWorkerHost(t.server, workerAddr, locs)
 	var buf bytes.Buffer
-	best, err := volume_replica.SyncAndSelectBestReplica(t.grpcDialOption, needle.VolumeId(t.volumeID), t.collection, locs, "", &buf)
+	best, err := volume_replica.SelectSourceIfIndexesMatch(t.grpcDialOption, needle.VolumeId(t.volumeID), t.collection, locs, prefer, &buf)
 	if out := strings.TrimSpace(buf.String()); out != "" {
-		glog.Infof("EC encode replica sync for volume %d:\n%s", t.volumeID, out)
+		glog.Infof("EC encode replica idx compare for volume %d:\n%s", t.volumeID, out)
 	}
 	if err != nil {
 		return err
 	}
 	if best.Url != "" && best.Url != t.server {
-		glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server)
+		glog.Infof("EC encode: late-binding source %s for volume %d (was %s)", best.Url, t.volumeID, t.server)
 		t.server = best.Url
 	}
-	// Late-bind from after consistency: worker -address hostname, not Sources[0]
-	// and not AddressIsLocal (replica URLs are hostnames like full-02:8081).
-	workerAddr := strings.TrimSpace(t.workerAddress)
-	if workerAddr == "" {
-		workerAddr, _ = os.Hostname()
-	}
-	if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server {
-		glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server)
-		t.server = local
-	}
 	return nil
 }
 
-type replicaFileCount struct {
-	url       string
-	fileCount uint64
-	err       error
-}
-
-// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica
-// only when live FileCounts are equal. Diverged counts keep current (max).
-func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string {
-	statuses := volume_replica.GetReplicaStatuses(dial, vid, locs)
-	replicas := make([]replicaFileCount, 0, len(statuses))
-	for _, s := range statuses {
-		replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error})
-	}
-	return lateBindSourceToWorkerHost(current, workerAddress, replicas)
-}
-
-// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host
-// (any port). Diverge → keep current (already max FileCount). Worker address
-// is not a volume URL.
-func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string {
+// lateBindSourceToWorkerHost returns the replica on the worker -address host
+// (any port), keeping current when no replica shares that host. Worker address
+// is not a volume URL. Safe to prefer after the idx gate: matching live sets
+// make every listed replica an equivalent source.
+func lateBindSourceToWorkerHost(current, workerAddress string, locs []wdclient.Location) string {
 	workerHost := util.HostKey(workerAddress)
 	if workerHost == "" {
 		return current
 	}
-	var live []replicaFileCount
-	for _, r := range replicas {
-		if r.err != nil || r.url == "" {
-			continue
-		}
-		live = append(live, r)
-	}
-	if len(live) == 0 {
-		return current
-	}
-	maxCount := live[0].fileCount
-	allSame := true
-	for _, r := range live[1:] {
-		if r.fileCount != live[0].fileCount {
-			allSame = false
-		}
-		if r.fileCount > maxCount {
-			maxCount = r.fileCount
-		}
-	}
-	if !allSame {
-		return current
-	}
-	for _, r := range live {
-		if util.HostKey(r.url) == workerHost {
-			return r.url
+	for _, l := range locs {
+		if util.HostKey(l.Url) == workerHost {
+			return l.Url
 		}
 	}
 	return current

weed/worker/tasks/erasure_coding/ec_task_encode_files_e2e_test.go

@@ -19,15 +19,10 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-// End-to-end on two servers: a volume with a real replica on server A and a
-// 0-byte stub replica of the same id on server B (an interrupted-encode
-// leftover). After a full EC encode the cluster must end in exactly one valid
-// layout — the complete shard set split across A and B, each with .ecx/.vif —
-// and every wrong file must be gone: both regular .dat files (source deleted
-// after verify, stub swept before distribute), no shard on the wrong server,
-// and B's shared <collection>_<vid>.vif intact rather than clobbered by the
-// stub's delete.
-func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) {
+// End-to-end on two servers: a real replica on A and a 0-byte stub on B
+// (interrupted-encode leftover). Empty live-idx vs non-empty must refuse
+// encode; originals stay, t.server is not rebound to the stub.
+func TestEcEncodeRefusesOnEmptyStubReplica(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
@@ -93,39 +88,16 @@ func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
 	defer cancel()
 
-	require.NoError(t, task.Execute(ctx, params))
+	err := task.Execute(ctx, params)
+	require.Error(t, err, "empty live-idx vs non-empty must refuse encode")
+	require.Contains(t, err.Error(), "no union")
+	require.NotEqual(t, addrB, task.server, "after gate t.server is not a stub URL")
 
 	dirA := filepath.Join(cluster.BaseDir(), "volume0")
 	dirB := filepath.Join(cluster.BaseDir(), "volume1")
 	base := fmt.Sprintf("%s_%d", collection, volumeID)
-
-	// Both original regular volumes are gone: A's source deleted after verify,
-	// B's stub swept before distribute.
-	requireAbsent(t, dirA, base+".dat")
-	requireAbsent(t, dirA, base+".idx")
-	requireAbsent(t, dirB, base+".dat")
-	requireAbsent(t, dirB, base+".idx")
-
-	// Each server holds exactly its assigned shards, plus index/info sidecars.
-	for _, id := range aShards {
-		requirePresent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
-	}
-	for _, id := range bShards {
-		requireAbsent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
-	}
-	requirePresent(t, dirA, base+".ecx")
-	requirePresent(t, dirA, base+".vif")
-
-	for _, id := range bShards {
-		requirePresent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
-	}
-	for _, id := range aShards {
-		requireAbsent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
-	}
-	requirePresent(t, dirB, base+".ecx")
-	// The shared .vif must survive on B: the stub was deleted before the EC
-	// files landed, so deleteOriginalVolume never ran removeVolumeFiles there.
-	requirePresent(t, dirB, base+".vif")
+	requirePresent(t, dirA, base+".dat")
+	requirePresent(t, dirB, base+".dat")
 }
 
 // serverAddress builds the SeaweedFS ip:httpPort.grpcPort address the worker's

weed/worker/tasks/erasure_coding/ec_task_idx_gate_test.go

@@ -0,0 +1,279 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestEcEncodeAbortsOnEqualFileCountDifferentIds(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster, task, params, ctx := twoReplicaTask(t, 9610, "ec-idx-mismatch")
+	httpClient := framework.NewHTTPClient()
+	const volumeID = uint32(9610)
+
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{1, 2, 3})
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{1, 2, 4})
+
+	require.NoError(t, task.markReplicasReadonly(ctx))
+	err := task.syncAndSelectSourceReplica()
+	require.Error(t, err, "equal FileCount with different ids must abort")
+	require.Contains(t, err.Error(), "no union")
+
+	execErr := task.Execute(ctx, params)
+	require.Error(t, execErr)
+	require.Contains(t, execErr.Error(), "no union")
+	require.NotContains(t, execErr.Error(), "building union")
+
+	dirA := cluster.VolumeDiskDir(0, 0)
+	dirB := cluster.VolumeDiskDir(1, 0)
+	base := fmt.Sprintf("%s_%d", "ec-idx-mismatch", volumeID)
+	requirePresent(t, dirA, base+".dat")
+	requirePresent(t, dirB, base+".dat")
+	require.True(t, idxHasNeedle(t, filepath.Join(dirA, base+".idx"), 3))
+	require.False(t, idxHasNeedle(t, filepath.Join(dirA, base+".idx"), 4))
+	require.True(t, idxHasNeedle(t, filepath.Join(dirB, base+".idx"), 4))
+	require.False(t, idxHasNeedle(t, filepath.Join(dirB, base+".idx"), 3))
+}
+
+func TestEcEncodeAllowsVacuumOffsetMismatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster, task, params, ctx := twoReplicaTask(t, 9611, "ec-idx-vacuum")
+	httpClient := framework.NewHTTPClient()
+	const volumeID = uint32(9611)
+
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{10, 11, 12})
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{10, 11, 12})
+
+	// Delete the first needle so vacuum packs 11/12 to new offsets on A only.
+	deletedFID := framework.NewFileID(volumeID, 10, cookieFor(10))
+	for _, url := range []string{cluster.VolumeAdminURL(0), cluster.VolumeAdminURL(1)} {
+		req, err := http.NewRequest(http.MethodDelete, url+"/"+deletedFID, nil)
+		require.NoError(t, err)
+		resp := framework.DoRequest(t, httpClient, req)
+		_ = framework.ReadAllAndClose(t, resp)
+		require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	}
+
+	connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer connA.Close()
+	compactVolumeOnce(t, clientA, volumeID)
+
+	offA := needleOffset(t, filepath.Join(cluster.VolumeDiskDir(0, 0), "ec-idx-vacuum_9611.idx"), 11)
+	offB := needleOffset(t, filepath.Join(cluster.VolumeDiskDir(1, 0), "ec-idx-vacuum_9611.idx"), 11)
+	require.NotEqual(t, offA, offB, "vacuum should rewrite offsets on one replica")
+
+	require.NoError(t, task.Execute(ctx, params))
+}
+
+func TestEcEncodeAbortsWhenIdxRpcFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster, task, _, ctx := twoReplicaTask(t, 9612, "ec-idx-rpc")
+	httpClient := framework.NewHTTPClient()
+	const volumeID = uint32(9612)
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{1})
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{1})
+
+	require.NoError(t, task.markReplicasReadonly(ctx))
+
+	locs := []wdclient.Location{
+		{Url: serverAddress(cluster, 0)},
+		{Url: "127.0.0.1:1"},
+	}
+	_, err := volume_replica.SelectSourceIfIndexesMatch(task.grpcDialOption, needle.VolumeId(volumeID), "ec-idx-rpc", locs, "", &bytes.Buffer{})
+	require.Error(t, err)
+}
+
+func TestEcEncodeSingleReplicaAfterReadonly(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(9613)
+	framework.AllocateVolume(t, grpcClient, volumeID, "ec-idx-one")
+	httpClient := framework.NewHTTPClient()
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(), volumeID, []uint64{7, 8})
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	task := NewErasureCodingTask("ec-idx-one", cluster.VolumeServerAddress(), volumeID, "ec-idx-one", dialOption)
+	params := singleReplicaParams(cluster.VolumeServerAddress(), volumeID, "ec-idx-one", t.TempDir())
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	require.NoError(t, task.markReplicasReadonly(ctx))
+	require.NoError(t, task.syncAndSelectSourceReplica())
+	require.NoError(t, task.Execute(ctx, params))
+}
+
+func TestHttpWriteAfterReadonlyIsNotCreated(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(9614)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+	httpClient := framework.NewHTTPClient()
+	uploadNeedles(t, httpClient, cluster.VolumeAdminURL(), volumeID, []uint64{1})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true})
+	require.NoError(t, err)
+
+	fid := framework.NewFileID(volumeID, 99, cookieFor(99))
+	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("must-not-land"))
+	_ = framework.ReadAllAndClose(t, resp)
+	require.NotEqual(t, http.StatusCreated, resp.StatusCode)
+	require.NotEqual(t, http.StatusNoContent, resp.StatusCode)
+
+	idxPath := filepath.Join(cluster.BaseDir(), "volume", fmt.Sprintf("%d.idx", volumeID))
+	require.False(t, idxHasNeedle(t, idxPath, 99))
+}
+
+func twoReplicaTask(t *testing.T, volumeID uint32, collection string) (*framework.MultiVolumeCluster, *ErasureCodingTask, *worker_pb.TaskParams, context.Context) {
+	t.Helper()
+	cluster := framework.StartMultiVolumeCluster(t, matrix.P1(), 2)
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	addrA := serverAddress(cluster, 0)
+	addrB := serverAddress(cluster, 1)
+
+	connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	t.Cleanup(func() { connA.Close() })
+	connB, clientB := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	t.Cleanup(func() { connB.Close() })
+	framework.AllocateVolume(t, clientA, volumeID, collection)
+	framework.AllocateVolume(t, clientB, volumeID, collection)
+
+	dataShards := int(erasure_coding.DataShardsCount)
+	totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount)
+	task := NewErasureCodingTask("ec-idx-gate", addrA, volumeID, collection, dialOption)
+	params := &worker_pb.TaskParams{
+		VolumeId:   volumeID,
+		Collection: collection,
+		Sources: []*worker_pb.TaskSource{
+			{Node: addrA, VolumeId: volumeID},
+			{Node: addrB, VolumeId: volumeID},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{Node: addrA, ShardIds: shardRange(0, dataShards)},
+			{Node: addrB, ShardIds: shardRange(dataShards, totalShards)},
+		},
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:   erasure_coding.DataShardsCount,
+				ParityShards: erasure_coding.ParityShardsCount,
+				WorkingDir:   t.TempDir(),
+			},
+		},
+	}
+	task.sources = params.Sources
+	task.targets = params.Targets
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	t.Cleanup(cancel)
+	return cluster, task, params, ctx
+}
+
+func singleReplicaParams(node string, volumeID uint32, collection, workDir string) *worker_pb.TaskParams {
+	totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount)
+	return &worker_pb.TaskParams{
+		VolumeId:   volumeID,
+		Collection: collection,
+		Sources:    []*worker_pb.TaskSource{{Node: node, VolumeId: volumeID}},
+		Targets:    []*worker_pb.TaskTarget{{Node: node, ShardIds: shardRange(0, totalShards)}},
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:   erasure_coding.DataShardsCount,
+				ParityShards: erasure_coding.ParityShardsCount,
+				WorkingDir:   workDir,
+			},
+		},
+	}
+}
+
+func cookieFor(id uint64) uint32 {
+	return 0xEC100000 + uint32(id)
+}
+
+func uploadNeedles(t *testing.T, client *http.Client, volumeURL string, volumeID uint32, ids []uint64) {
+	t.Helper()
+	for _, id := range ids {
+		fid := framework.NewFileID(volumeID, id, cookieFor(id))
+		resp := framework.UploadBytes(t, client, volumeURL, fid, []byte(fmt.Sprintf("payload-%d", id)))
+		_ = framework.ReadAllAndClose(t, resp)
+		require.Equal(t, http.StatusCreated, resp.StatusCode)
+	}
+}
+
+func idxHasNeedle(t *testing.T, idxPath string, id uint64) bool {
+	t.Helper()
+	f, err := os.Open(idxPath)
+	require.NoError(t, err)
+	defer f.Close()
+	want := types.NeedleId(id)
+	found := false
+	err = idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		if key == want && !offset.IsZero() && !size.IsDeleted() {
+			found = true
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	return found
+}
+
+func needleOffset(t *testing.T, idxPath string, id uint64) types.Offset {
+	t.Helper()
+	f, err := os.Open(idxPath)
+	require.NoError(t, err)
+	defer f.Close()
+	want := types.NeedleId(id)
+	var off types.Offset
+	var seen bool
+	err = idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		if key == want && !offset.IsZero() && !size.IsDeleted() {
+			off = offset
+			seen = true
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	require.True(t, seen, "needle %d missing in %s", id, idxPath)
+	return off
+}

weed/worker/tasks/erasure_coding/ec_task_julor_layout_e2e_test.go

@@ -2,10 +2,7 @@ package erasure_coding
 
 import (
 	"context"
-	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -19,20 +16,10 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-// Reproduces the issue-9478 volume-13 layout end to end on a multi-server,
-// multi-disk cluster and asserts the cluster converges to exactly one valid EC
-// layout. Before the encode the volume is in the stuck state: a real .dat
-// source, a 0-byte stub replica left by an interrupted encode, and partial
-// stale EC shards from that attempt. After the worker runs (sweep stubs ->
-// clear stale shards -> distribute -> mount -> verify -> delete originals):
-//
-//   - every assigned shard is present on its target disk, with .ecx/.vif
-//     alongside it (the EC volume's info file survives even when shards land on
-//     the source's own disk);
-//   - no regular .dat/.idx remains anywhere — the source is deleted after
-//     verify, the stub was swept before distribute;
-//   - the stale shards that aren't part of the new assignment are gone.
-func TestEcEncodeJulorLayoutConverges(t *testing.T) {
+// A real .dat source plus a 0-byte stub replica left by an interrupted encode
+// and partial stale EC shards: empty live-idx vs non-empty must refuse encode
+// (no union, no skip).
+func TestEcEncodeJulorLayoutRefuses(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
@@ -122,52 +109,8 @@ func TestEcEncodeJulorLayoutConverges(t *testing.T) {
 		},
 	}
 
-	require.NoError(t, task.Execute(ctx, params))
-
-	base := fmt.Sprintf("%s_%d", collection, volumeID)
-
-	// Shards are distributed per-server; the disk within a server is auto-selected
-	// when DiskId is 0, and on a multi-disk server shards can spread to a sibling
-	// disk while the .ecx/.vif index lives on one of them (the cross-disk EC
-	// layout). So assert per server (aggregate across its disks): each server
-	// holds exactly its assigned shards, its .ecx/.vif survive on at least one
-	// disk (the info file is not stripped, even on the source's own disk), and no
-	// regular .dat/.idx remains. Stale shards 11-13 mounted on the source must be
-	// gone — they are not in S0's assignment.
-	serverShards := map[int][]uint32{
-		0: shardRange(0, 5),   // S0 (source): 0..4
-		1: shardRange(5, 10),  // S1 (stub):   5..9
-		2: shardRange(10, 14), // S2 (target): 10..13
-	}
-	allShards := map[uint32]bool{}
-	for server, want := range serverShards {
-		found := map[uint32]bool{}
-		hasEcx, hasVif := false, false
-		for d := 0; d < disksPerServer; d++ {
-			dir := cluster.VolumeDiskDir(server, d)
-			requireAbsent(t, dir, base+".dat")
-			requireAbsent(t, dir, base+".idx")
-			for id := 0; id < erasure_coding.TotalShardsCount; id++ {
-				if ecFileExists(dir, fmt.Sprintf("%s.ec%02d", base, id)) {
-					found[uint32(id)] = true
-					allShards[uint32(id)] = true
-				}
-			}
-			hasEcx = hasEcx || ecFileExists(dir, base+".ecx")
-			hasVif = hasVif || ecFileExists(dir, base+".vif")
-		}
-		gotIDs := make([]uint32, 0, len(found))
-		for id := range found {
-			gotIDs = append(gotIDs, id)
-		}
-		require.ElementsMatch(t, want, gotIDs, "server %d should hold exactly its assigned shards", server)
-		require.True(t, hasEcx, "server %d must keep its .ecx", server)
-		require.True(t, hasVif, "server %d must keep its .vif (not stripped by the original-volume delete)", server)
-	}
-	require.Len(t, allShards, erasure_coding.TotalShardsCount, "the full shard set must be present across the cluster")
-}
-
-func ecFileExists(dir, name string) bool {
-	_, err := os.Stat(filepath.Join(dir, name))
-	return err == nil
+	err = task.Execute(ctx, params)
+	require.Error(t, err, "empty live-idx vs non-empty must refuse encode")
+	require.Contains(t, err.Error(), "no union")
+	require.NotEqual(t, addr(stubServer), task.server, "after gate t.server is not a stub URL")
 }

weed/worker/tasks/erasure_coding/ec_task_source_host_test.go

@@ -1,31 +1,34 @@
 package erasure_coding
 
-import "testing"
+import (
+	"testing"
 
-func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) {
-	a := replicaFileCount{url: "full-01:8080", fileCount: 21}
-	b := replicaFileCount{url: "full-02:8081", fileCount: 21}
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+func TestLateBindSourceToWorkerHostPrefersWorkerHost(t *testing.T) {
+	a := wdclient.Location{Url: "full-01:8080"}
+	b := wdclient.Location{Url: "full-02:8081"}
 	worker := "full-02:9334"
 
 	// Sources [A,B] — topology-first is A; worker on B must still copy B.
-	if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url {
-		t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url)
+	if got := lateBindSourceToWorkerHost(a.Url, worker, []wdclient.Location{a, b}); got != b.Url {
+		t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.Url)
 	}
 	// Invert Sources — still B.
-	if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url {
-		t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url)
+	if got := lateBindSourceToWorkerHost(b.Url, worker, []wdclient.Location{b, a}); got != b.Url {
+		t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.Url)
 	}
 }
 
-func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) {
-	a := replicaFileCount{url: "full-01:8080", fileCount: 40}
-	b := replicaFileCount{url: "full-02:8081", fileCount: 21}
-	worker := "full-02:9334"
+func TestLateBindSourceToWorkerHostNoMatchKeepsCurrent(t *testing.T) {
+	a := wdclient.Location{Url: "full-01:8080"}
+	b := wdclient.Location{Url: "full-02:8081"}
 
-	if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url {
-		t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url)
+	if got := lateBindSourceToWorkerHost(a.Url, "other-host:9334", []wdclient.Location{a, b}); got != a.Url {
+		t.Fatalf("no worker-host replica: got %s want %s", got, a.Url)
 	}
-	if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url {
-		t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url)
+	if got := lateBindSourceToWorkerHost(a.Url, "", []wdclient.Location{a, b}); got != a.Url {
+		t.Fatalf("empty worker address: got %s want %s", got, a.Url)
 	}
 }