diff --git a/weed/ec/ec_encode.go b/weed/ec/ec_encode.go index 75f4a73ad..765810af0 100644 --- a/weed/ec/ec_encode.go +++ b/weed/ec/ec_encode.go @@ -327,17 +327,21 @@ func doEcEncode(env *Env, writer io.Writer, volumeIdToCollection map[needle.Volu return nil, err } - // Sync replicas and select the best one for each volume (with highest file count) - // This addresses data inconsistency risk in multi-replica volumes (issue #7797) - // by syncing missing entries between replicas before encoding + // Compare live .idx sets across all replicas, not only the capacity + // subset — a filtered-out replica with a divergent live set still gets + // deleted below, so it must refuse encode here. The generation host + // must come from filteredLocations. bestReplicas := make(map[needle.VolumeId]wdclient.Location) for _, vid := range volumeIds { collection := volumeIdToCollection[vid] - - // Sync missing entries between replicas, then select the best one - bestLoc, selectErr := volume_replica.SyncAndSelectBestReplica(env.GrpcDialOption, vid, collection, filteredLocations[vid], "", writer) + prefer := filteredLocations[vid][0].Url + bestLoc, selectErr := volume_replica.SelectSourceIfIndexesMatch(env.GrpcDialOption, vid, collection, locations[vid], prefer, writer) if selectErr != nil { - return nil, fmt.Errorf("failed to sync and select replica for volume %d: %v", vid, selectErr) + return nil, fmt.Errorf("failed to compare replica indexes for volume %d: %v", vid, selectErr) + } + bestLoc, capErr := volume_replica.RequireCapacitySource(vid, bestLoc, filteredLocations[vid]) + if capErr != nil { + return nil, capErr } bestReplicas[vid] = bestLoc } diff --git a/weed/storage/store.go b/weed/storage/store.go index a7a54091c..06c87c823 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -900,10 +900,14 @@ func (s *Store) MarkVolumeReadonly(i needle.VolumeId, canDelete bool, persist bo if v == nil { return fmt.Errorf("volume %d not found", i) } + // Wait for in-flight appends so a mark acked mid-write cannot race an + // encode that reads the index and deletes the replica the write lands on. + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() if canDelete && !v.HasRemoteFile() { // deletes append tombstones to .idx, which a readonly boot opened // O_RDONLY; remote volumes already delete through a RDWR idx - if err := v.reopenIdxForWrite(); err != nil { + if err := v.reopenIdxForWriteLocked(); err != nil { return fmt.Errorf("volume %d reopen idx for write: %v", i, err) } } diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index dc4ffa365..a3e5bbd6a 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -65,7 +65,12 @@ func loadVolumeWithoutWorker(dirname string, dirIdx string, collection string, i func (v *Volume) reopenIdxForWrite() error { v.dataFileAccessLock.Lock() defer v.dataFileAccessLock.Unlock() + return v.reopenIdxForWriteLocked() +} +// reopenIdxForWriteLocked is reopenIdxForWrite for callers already holding +// dataFileAccessLock (MarkVolumeReadonly). +func (v *Volume) reopenIdxForWriteLocked() error { oldNm, isSorted := v.nm.(*SortedFileNeedleMap) if !isSorted { return nil diff --git a/weed/storage/volume_replica/idx_set.go b/weed/storage/volume_replica/idx_set.go new file mode 100644 index 000000000..c26c36ee4 --- /dev/null +++ b/weed/storage/volume_replica/idx_set.go @@ -0,0 +1,145 @@ +package volume_replica + +import ( + "fmt" + "io" + + "google.golang.org/grpc" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/wdclient" +) + +type replicaLiveIdx struct { + loc wdclient.Location + live []types.NeedleId +} + +// LiveNeedlesFromIndex walks a compact needle map and returns live NeedleIds +// in key order. Deleted, invalid-size, and zero-offset rows are skipped; Size +// is not part of the comparison since replicas can differ by MIME/flags while +// holding the same ids. +func LiveNeedlesFromIndex(db *needle_map.MemDb) ([]types.NeedleId, error) { + if db == nil { + return nil, fmt.Errorf("nil needle index") + } + var live []types.NeedleId + err := db.AscendingVisit(func(nv needle_map.NeedleValue) error { + if nv.Offset.IsZero() || nv.Size.IsDeleted() || !nv.Size.IsValid() { + return nil + } + live = append(live, nv.Key) + return nil + }) + return live, err +} + +// EqualLiveNeedleSets reports whether two live NeedleId sets match. +func EqualLiveNeedleSets(a, b []types.NeedleId) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// RequireCapacitySource returns loc if it is in capacityLocs (FreeVolumeCount>=2). +func RequireCapacitySource(vid needle.VolumeId, loc wdclient.Location, capacityLocs []wdclient.Location) (wdclient.Location, error) { + if loc.Url == "" { + return wdclient.Location{}, fmt.Errorf("volume %d: no encode source after idx compare", vid) + } + for _, c := range capacityLocs { + if c.Url == loc.Url { + return loc, nil + } + } + return wdclient.Location{}, fmt.Errorf("volume %d: selected source %s is not in the FreeVolumeCount>=2 replica set; refusing encode", vid, loc.Url) +} + +// SelectSourceIfIndexesMatch copies each replica's live .idx and compares +// NeedleId sets. A read error or a mismatch aborts. One replica skips the +// comparison. On match, the source is prefer when listed, else the first. +func SelectSourceIfIndexesMatch(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, locations []wdclient.Location, prefer string, writer io.Writer) (wdclient.Location, error) { + if len(locations) == 0 { + return wdclient.Location{}, fmt.Errorf("no replicas available for volume %d", vid) + } + if len(locations) == 1 { + if writer != nil { + fmt.Fprintf(writer, "volume %d: single replica %s, skipping idx set compare\n", vid, locations[0].Url) + } + return locations[0], nil + } + + views := make([]replicaLiveIdx, 0, len(locations)) + for _, loc := range locations { + status := GetReplicaStatus(grpcDialOption, vid, loc) + if status.Error != nil { + return wdclient.Location{}, fmt.Errorf("volume %d: replica status %s: %w", vid, loc.Url, status.Error) + } + db, err := ReadReplicaIndex(grpcDialOption, vid, collection, loc.ServerAddress()) + if err != nil { + return wdclient.Location{}, fmt.Errorf("volume %d: read .idx from %s: %w", vid, loc.Url, err) + } + live, err := LiveNeedlesFromIndex(db) + db.Close() + if err != nil { + return wdclient.Location{}, fmt.Errorf("volume %d: walk .idx from %s: %w", vid, loc.Url, err) + } + views = append(views, replicaLiveIdx{loc: loc, live: live}) + } + + return chooseSourceFromLiveNeedles(vid, views, prefer, writer) +} + +func chooseSourceFromLiveNeedles(vid needle.VolumeId, views []replicaLiveIdx, prefer string, writer io.Writer) (wdclient.Location, error) { + if len(views) == 0 { + return wdclient.Location{}, fmt.Errorf("volume %d: no data-bearing replicas to encode", vid) + } + + reference := views[0].live + referenceUrl := views[0].loc.Url + for _, view := range views[1:] { + if !EqualLiveNeedleSets(reference, view.live) { + return wdclient.Location{}, fmt.Errorf("volume %d: replica %s live NeedleId set does not match %s (%d vs %d entries); refusing encode (no union)", + vid, view.loc.Url, referenceUrl, len(view.live), len(reference)) + } + } + + if len(reference) == 0 { + return wdclient.Location{}, fmt.Errorf("volume %d: no data-bearing replicas to encode", vid) + } + + if writer != nil { + fmt.Fprintf(writer, "volume %d: all %d data replicas are consistent (matching live idx sets, %d needles)\n", + vid, len(views), len(reference)) + } + + if prefer != "" { + for _, view := range views { + if view.loc.Url == prefer { + return view.loc, nil + } + } + } + return views[0].loc, nil +} + +// ReadReplicaIndex streams a replica's .idx into an in-memory compact map. +func ReadReplicaIndex(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, server pb.ServerAddress) (*needle_map.MemDb, error) { + db := needle_map.NewMemDb() + if db == nil { + return nil, fmt.Errorf("failed to allocate in-memory needle DB") + } + if err := loadReplicaIndex(db, grpcDialOption, vid, collection, server); err != nil { + db.Close() + return nil, err + } + return db, nil +} diff --git a/weed/storage/volume_replica/idx_set_test.go b/weed/storage/volume_replica/idx_set_test.go new file mode 100644 index 000000000..01de1f9a8 --- /dev/null +++ b/weed/storage/volume_replica/idx_set_test.go @@ -0,0 +1,195 @@ +package volume_replica + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/wdclient" +) + +func TestEqualLiveNeedleSets_SameFileCountDifferentIds(t *testing.T) { + a := needle_map.NewMemDb() + b := needle_map.NewMemDb() + defer a.Close() + defer b.Close() + + // Equal cardinality, different ids — FileCount would match, live sets must not. + a.Set(types.NeedleId(1), types.ToOffset(8), types.Size(100)) + a.Set(types.NeedleId(2), types.ToOffset(16), types.Size(100)) + a.Set(types.NeedleId(3), types.ToOffset(24), types.Size(100)) + b.Set(types.NeedleId(1), types.ToOffset(8), types.Size(100)) + b.Set(types.NeedleId(2), types.ToOffset(16), types.Size(100)) + b.Set(types.NeedleId(4), types.ToOffset(24), types.Size(100)) + + la, err := LiveNeedlesFromIndex(a) + if err != nil { + t.Fatal(err) + } + lb, err := LiveNeedlesFromIndex(b) + if err != nil { + t.Fatal(err) + } + if EqualLiveNeedleSets(la, lb) { + t.Fatalf("equal FileCount with different ids must not match: a=%v b=%v", la, lb) + } +} + +func TestEqualLiveNeedleSets_VacuumOffsetsIgnored(t *testing.T) { + a := needle_map.NewMemDb() + b := needle_map.NewMemDb() + defer a.Close() + defer b.Close() + + a.Set(types.NeedleId(10), types.ToOffset(8), types.Size(50)) + a.Set(types.NeedleId(11), types.ToOffset(64), types.Size(80)) + // Vacuum rewrote the same live needles to new offsets. + b.Set(types.NeedleId(10), types.ToOffset(8), types.Size(50)) + b.Set(types.NeedleId(11), types.ToOffset(200), types.Size(80)) + + la, err := LiveNeedlesFromIndex(a) + if err != nil { + t.Fatal(err) + } + lb, err := LiveNeedlesFromIndex(b) + if err != nil { + t.Fatal(err) + } + if !EqualLiveNeedleSets(la, lb) { + t.Fatalf("same ids with different offsets must match: a=%v b=%v", la, lb) + } +} + +func TestEqualLiveNeedleSets_SizeMismatchIgnored(t *testing.T) { + a := needle_map.NewMemDb() + b := needle_map.NewMemDb() + defer a.Close() + defer b.Close() + + a.Set(types.NeedleId(10), types.ToOffset(8), types.Size(50)) + b.Set(types.NeedleId(10), types.ToOffset(8), types.Size(74)) + + la, err := LiveNeedlesFromIndex(a) + if err != nil { + t.Fatal(err) + } + lb, err := LiveNeedlesFromIndex(b) + if err != nil { + t.Fatal(err) + } + if !EqualLiveNeedleSets(la, lb) { + t.Fatalf("same NeedleId with different Size must match: a=%v b=%v", la, lb) + } +} + +func TestLiveNeedlesFromIndex_SkipsDeleted(t *testing.T) { + db := needle_map.NewMemDb() + defer db.Close() + db.Set(types.NeedleId(1), types.ToOffset(8), types.Size(40)) + db.Set(types.NeedleId(2), types.ToOffset(16), types.TombstoneFileSize) + db.Set(types.NeedleId(3), types.ToOffset(24), types.Size(0)) + + live, err := LiveNeedlesFromIndex(db) + if err != nil { + t.Fatal(err) + } + if len(live) != 1 || live[0] != 1 { + t.Fatalf("live=%v, want only id=1", live) + } +} + +func TestSelectSourceIfIndexesMatch_SingleReplica(t *testing.T) { + loc := wdclient.Location{Url: "127.0.0.1:8080"} + got, err := SelectSourceIfIndexesMatch(nil, needle.VolumeId(7), "", []wdclient.Location{loc}, "other", io.Discard) + if err != nil { + t.Fatal(err) + } + if got.Url != loc.Url { + t.Fatalf("got %+v, want %+v", got, loc) + } +} + +func TestSelectSourceIfIndexesMatch_NoReplicas(t *testing.T) { + _, err := SelectSourceIfIndexesMatch(nil, needle.VolumeId(7), "", nil, "", io.Discard) + if err == nil { + t.Fatal("expected error for empty replica list") + } +} + +func TestChooseSourceFromLiveNeedles_SuccessContainsConsistent(t *testing.T) { + a := wdclient.Location{Url: "127.0.0.1:8080"} + b := wdclient.Location{Url: "127.0.0.1:8081"} + live := []types.NeedleId{1, 2} + var buf bytes.Buffer + got, err := chooseSourceFromLiveNeedles(7, []replicaLiveIdx{ + {loc: a, live: live}, + {loc: b, live: append([]types.NeedleId(nil), live...)}, + }, "", &buf) + if err != nil { + t.Fatal(err) + } + if got.Url != a.Url { + t.Fatalf("got %+v, want %+v", got, a) + } + if !strings.Contains(buf.String(), "replicas are consistent") { + t.Fatalf("success log %q must contain %q", buf.String(), "replicas are consistent") + } + wantLine := "volume 7: all 2 data replicas are consistent (matching live idx sets, 2 needles)" + if !strings.Contains(buf.String(), wantLine) { + t.Fatalf("success log %q must contain %q", buf.String(), wantLine) + } +} + +func TestChooseSourceFromLiveNeedles_EmptyVsLiveRefuses(t *testing.T) { + a := wdclient.Location{Url: "127.0.0.1:8080"} + b := wdclient.Location{Url: "127.0.0.1:8081"} + _, err := chooseSourceFromLiveNeedles(7, []replicaLiveIdx{ + {loc: a, live: []types.NeedleId{1}}, + {loc: b, live: nil}, // empty / superblock stub + }, "", io.Discard) + if err == nil { + t.Fatal("empty live-idx vs non-empty must error, not skip") + } + if strings.Contains(err.Error(), "skipping") { + t.Fatalf("must refuse, not skip: %v", err) + } +} + +func TestChooseSourceFromLiveNeedles_AllEmpty(t *testing.T) { + a := wdclient.Location{Url: "127.0.0.1:8080"} + b := wdclient.Location{Url: "127.0.0.1:8081"} + _, err := chooseSourceFromLiveNeedles(7, []replicaLiveIdx{ + {loc: a, live: nil}, + {loc: b, live: nil}, + }, "", io.Discard) + if err == nil { + t.Fatal("all empty live-idx must error") + } + if !strings.Contains(err.Error(), "no data-bearing replicas") { + t.Fatalf("got %v, want no data-bearing replicas", err) + } +} + +func TestRequireCapacitySource_ThreeLocationsSourceFromCapacity(t *testing.T) { + fat := wdclient.Location{Url: "fat:8080"} + cap1 := wdclient.Location{Url: "cap1:8080"} + cap2 := wdclient.Location{Url: "cap2:8080"} + filtered := []wdclient.Location{cap1, cap2} + + got, err := RequireCapacitySource(1, cap1, filtered) + if err != nil { + t.Fatal(err) + } + if got.Url != cap1.Url { + t.Fatalf("got %+v, want capacity source %+v", got, cap1) + } + + _, err = RequireCapacitySource(1, fat, filtered) + if err == nil { + t.Fatal("must refuse encode from first fat replica") + } +} diff --git a/weed/storage/volume_replica/replica_sync.go b/weed/storage/volume_replica/replica_sync.go index 3997569c8..9a578ca8d 100644 --- a/weed/storage/volume_replica/replica_sync.go +++ b/weed/storage/volume_replica/replica_sync.go @@ -224,15 +224,19 @@ func (rub *unionBuilder) buildUnionReplica(locations []wdclient.Location, status } func (rub *unionBuilder) readIndexDatabase(db *needle_map.MemDb, server pb.ServerAddress) error { + return loadReplicaIndex(db, rub.grpcDialOption, rub.vid, rub.collection, server) +} + +func loadReplicaIndex(db *needle_map.MemDb, grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, server pb.ServerAddress) error { var buf bytes.Buffer - err := operation.WithVolumeServerClient(true, server, rub.grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(true, server, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{ - VolumeId: uint32(rub.vid), + VolumeId: uint32(vid), Ext: ".idx", CompactionRevision: math.MaxUint32, StopOffset: math.MaxInt64, - Collection: rub.collection, + Collection: collection, IsEcVolume: false, IgnoreSourceFileNotFound: false, }) diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index 6d657dfca..d90af4e92 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -380,6 +380,11 @@ func (v *Volume) writeNeedle2(n *needle.Needle, checkCookie bool, fsync bool, is func (v *Volume) doWriteRequest(n *needle.Needle, checkCookie bool, fsync bool) (offset uint64, size Size, isUnchanged bool, err error) { // glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String()) + // Re-check under the write lock: the outer checks run before the needle + // is assembled, and VolumeMarkReadonly can land in between. + if v.IsReadOnly() { + return 0, 0, false, fmt.Errorf("volume %d is read only", v.Id) + } if v.isFileUnchanged(n) { size = Size(n.DataSize) isUnchanged = true @@ -715,6 +720,9 @@ func (v *Volume) WriteNeedleBlob(needleId NeedleId, needleBlob []byte, size Size } } } + if v.IsReadOnly() { + return fmt.Errorf("volume %d is read only", v.Id) + } appendAtNs := needle.GetAppendAtNs(v.lastAppendAtNs) offset, err := needle.WriteNeedleBlob(v.DataBackend, needleBlob, size, appendAtNs, v.Version()) diff --git a/weed/storage/volume_write_test.go b/weed/storage/volume_write_test.go index f6e8b5270..d35cdda2b 100644 --- a/weed/storage/volume_write_test.go +++ b/weed/storage/volume_write_test.go @@ -447,3 +447,53 @@ func TestWriteNeedleBlobRoundTrip(t *testing.T) { t.Error("needle 4 read back with different data") } } + +// A write that already assembled the needle and holds the volume write lock +// must still refuse append once the volume is readonly — no 201/204, nothing +// in .dat/.idx. +func TestDoWriteRequestRefusesReadOnlyBeforeAppend(t *testing.T) { + dir := t.TempDir() + v, err := NewVolume(dir, dir, "", 21, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("volume creation: %v", err) + } + defer v.Close() + + seed := newRandomNeedle(1) + if _, _, _, err = v.writeNeedle2(seed, true, false, false); err != nil { + t.Fatalf("seed write: %v", err) + } + datBefore, _, err := v.DataBackend.GetStat() + if err != nil { + t.Fatalf("stat dat: %v", err) + } + idxBefore := v.nm.IndexFileSize() + + n := newRandomNeedle(99) + v.dataFileAccessLock.Lock() + v.noWriteLock.Lock() + v.noWriteOrDelete = true + v.noWriteLock.Unlock() + _, _, isUnchanged, writeErr := v.doWriteRequest(n, true, false) + v.dataFileAccessLock.Unlock() + + if writeErr == nil { + t.Fatal("expected read-only error") + } + if isUnchanged { + t.Fatal("read-only write must not report unchanged (HTTP 204)") + } + if _, found := v.nm.Get(n.Id); found { + t.Fatal("needle must not land in .idx") + } + datAfter, _, err := v.DataBackend.GetStat() + if err != nil { + t.Fatalf("stat dat after: %v", err) + } + if datAfter != datBefore { + t.Fatalf("dat size %d -> %d; needle must not append", datBefore, datAfter) + } + if v.nm.IndexFileSize() != idxBefore { + t.Fatalf("idx size changed; needle must not index") + } +} diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go index 043ff8677..f2838ffc8 100644 --- a/weed/worker/tasks/erasure_coding/ec_task.go +++ b/weed/worker/tasks/erasure_coding/ec_task.go @@ -181,11 +181,9 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP return fmt.Errorf("EC preflight failed for volume %d: %w", t.volumeID, err) } - // Step 1: Mark all replicas readonly, then reconcile them and select the most - // complete replica as the encode source. Encoding a stale replica and then - // deleting the originals would silently lose entries that exist only on another - // replica; SyncAndSelectBestReplica builds the union onto the best replica first - // (mirrors the shell ec.encode best-replica selection). + // Step 1: Mark all replicas readonly, then compare live .idx NeedleId + // sets. A mismatch or idx read error aborts encode and does not build a union. + // The encode source is late-bound only when the sets match. t.ReportProgressWithStage(10.0, "Marking volume readonly") t.GetLogger().Info("Marking volume readonly") if err := t.markReplicasReadonly(ctx); err != nil { @@ -391,90 +389,43 @@ func (t *ErasureCodingTask) markReplicasReadonly(ctx context.Context) error { return nil } -// syncAndSelectSourceReplica reconciles the volume's replicas (building the union -// of all live entries onto the most complete one) and switches the encode source -// to that replica, so a stale replica is never the basis of the encode. +// syncAndSelectSourceReplica compares live .idx sets after every replica is +// readonly. Matching sets late-bind the encode source; a mismatch or idx RPC +// error aborts without unioning replicas. func (t *ErasureCodingTask) syncAndSelectSourceReplica() error { locs := t.replicaLocations() - if len(locs) <= 1 { - return nil // single replica: nothing to reconcile + workerAddr := strings.TrimSpace(t.workerAddress) + if workerAddr == "" { + workerAddr, _ = os.Hostname() } + prefer := lateBindSourceToWorkerHost(t.server, workerAddr, locs) var buf bytes.Buffer - best, err := volume_replica.SyncAndSelectBestReplica(t.grpcDialOption, needle.VolumeId(t.volumeID), t.collection, locs, "", &buf) + best, err := volume_replica.SelectSourceIfIndexesMatch(t.grpcDialOption, needle.VolumeId(t.volumeID), t.collection, locs, prefer, &buf) if out := strings.TrimSpace(buf.String()); out != "" { - glog.Infof("EC encode replica sync for volume %d:\n%s", t.volumeID, out) + glog.Infof("EC encode replica idx compare for volume %d:\n%s", t.volumeID, out) } if err != nil { return err } if best.Url != "" && best.Url != t.server { - glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server) + glog.Infof("EC encode: late-binding source %s for volume %d (was %s)", best.Url, t.volumeID, t.server) t.server = best.Url } - // Late-bind from after consistency: worker -address hostname, not Sources[0] - // and not AddressIsLocal (replica URLs are hostnames like full-02:8081). - workerAddr := strings.TrimSpace(t.workerAddress) - if workerAddr == "" { - workerAddr, _ = os.Hostname() - } - if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server { - glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server) - t.server = local - } return nil } -type replicaFileCount struct { - url string - fileCount uint64 - err error -} - -// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica -// only when live FileCounts are equal. Diverged counts keep current (max). -func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string { - statuses := volume_replica.GetReplicaStatuses(dial, vid, locs) - replicas := make([]replicaFileCount, 0, len(statuses)) - for _, s := range statuses { - replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error}) - } - return lateBindSourceToWorkerHost(current, workerAddress, replicas) -} - -// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host -// (any port). Diverge → keep current (already max FileCount). Worker address -// is not a volume URL. -func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string { +// lateBindSourceToWorkerHost returns the replica on the worker -address host +// (any port), keeping current when no replica shares that host. Worker address +// is not a volume URL. Safe to prefer after the idx gate: matching live sets +// make every listed replica an equivalent source. +func lateBindSourceToWorkerHost(current, workerAddress string, locs []wdclient.Location) string { workerHost := util.HostKey(workerAddress) if workerHost == "" { return current } - var live []replicaFileCount - for _, r := range replicas { - if r.err != nil || r.url == "" { - continue - } - live = append(live, r) - } - if len(live) == 0 { - return current - } - maxCount := live[0].fileCount - allSame := true - for _, r := range live[1:] { - if r.fileCount != live[0].fileCount { - allSame = false - } - if r.fileCount > maxCount { - maxCount = r.fileCount - } - } - if !allSame { - return current - } - for _, r := range live { - if util.HostKey(r.url) == workerHost { - return r.url + for _, l := range locs { + if util.HostKey(l.Url) == workerHost { + return l.Url } } return current diff --git a/weed/worker/tasks/erasure_coding/ec_task_encode_files_e2e_test.go b/weed/worker/tasks/erasure_coding/ec_task_encode_files_e2e_test.go index baf6b42a0..2a6495ea7 100644 --- a/weed/worker/tasks/erasure_coding/ec_task_encode_files_e2e_test.go +++ b/weed/worker/tasks/erasure_coding/ec_task_encode_files_e2e_test.go @@ -19,15 +19,10 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// End-to-end on two servers: a volume with a real replica on server A and a -// 0-byte stub replica of the same id on server B (an interrupted-encode -// leftover). After a full EC encode the cluster must end in exactly one valid -// layout — the complete shard set split across A and B, each with .ecx/.vif — -// and every wrong file must be gone: both regular .dat files (source deleted -// after verify, stub swept before distribute), no shard on the wrong server, -// and B's shared _.vif intact rather than clobbered by the -// stub's delete. -func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) { +// End-to-end on two servers: a real replica on A and a 0-byte stub on B +// (interrupted-encode leftover). Empty live-idx vs non-empty must refuse +// encode; originals stay, t.server is not rebound to the stub. +func TestEcEncodeRefusesOnEmptyStubReplica(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } @@ -93,39 +88,16 @@ func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() - require.NoError(t, task.Execute(ctx, params)) + err := task.Execute(ctx, params) + require.Error(t, err, "empty live-idx vs non-empty must refuse encode") + require.Contains(t, err.Error(), "no union") + require.NotEqual(t, addrB, task.server, "after gate t.server is not a stub URL") dirA := filepath.Join(cluster.BaseDir(), "volume0") dirB := filepath.Join(cluster.BaseDir(), "volume1") base := fmt.Sprintf("%s_%d", collection, volumeID) - - // Both original regular volumes are gone: A's source deleted after verify, - // B's stub swept before distribute. - requireAbsent(t, dirA, base+".dat") - requireAbsent(t, dirA, base+".idx") - requireAbsent(t, dirB, base+".dat") - requireAbsent(t, dirB, base+".idx") - - // Each server holds exactly its assigned shards, plus index/info sidecars. - for _, id := range aShards { - requirePresent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id)) - } - for _, id := range bShards { - requireAbsent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id)) - } - requirePresent(t, dirA, base+".ecx") - requirePresent(t, dirA, base+".vif") - - for _, id := range bShards { - requirePresent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id)) - } - for _, id := range aShards { - requireAbsent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id)) - } - requirePresent(t, dirB, base+".ecx") - // The shared .vif must survive on B: the stub was deleted before the EC - // files landed, so deleteOriginalVolume never ran removeVolumeFiles there. - requirePresent(t, dirB, base+".vif") + requirePresent(t, dirA, base+".dat") + requirePresent(t, dirB, base+".dat") } // serverAddress builds the SeaweedFS ip:httpPort.grpcPort address the worker's diff --git a/weed/worker/tasks/erasure_coding/ec_task_idx_gate_test.go b/weed/worker/tasks/erasure_coding/ec_task_idx_gate_test.go new file mode 100644 index 000000000..4273b2990 --- /dev/null +++ b/weed/worker/tasks/erasure_coding/ec_task_idx_gate_test.go @@ -0,0 +1,279 @@ +package erasure_coding + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/idx" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_replica" + "github.com/seaweedfs/seaweedfs/weed/wdclient" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func TestEcEncodeAbortsOnEqualFileCountDifferentIds(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster, task, params, ctx := twoReplicaTask(t, 9610, "ec-idx-mismatch") + httpClient := framework.NewHTTPClient() + const volumeID = uint32(9610) + + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{1, 2, 3}) + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{1, 2, 4}) + + require.NoError(t, task.markReplicasReadonly(ctx)) + err := task.syncAndSelectSourceReplica() + require.Error(t, err, "equal FileCount with different ids must abort") + require.Contains(t, err.Error(), "no union") + + execErr := task.Execute(ctx, params) + require.Error(t, execErr) + require.Contains(t, execErr.Error(), "no union") + require.NotContains(t, execErr.Error(), "building union") + + dirA := cluster.VolumeDiskDir(0, 0) + dirB := cluster.VolumeDiskDir(1, 0) + base := fmt.Sprintf("%s_%d", "ec-idx-mismatch", volumeID) + requirePresent(t, dirA, base+".dat") + requirePresent(t, dirB, base+".dat") + require.True(t, idxHasNeedle(t, filepath.Join(dirA, base+".idx"), 3)) + require.False(t, idxHasNeedle(t, filepath.Join(dirA, base+".idx"), 4)) + require.True(t, idxHasNeedle(t, filepath.Join(dirB, base+".idx"), 4)) + require.False(t, idxHasNeedle(t, filepath.Join(dirB, base+".idx"), 3)) +} + +func TestEcEncodeAllowsVacuumOffsetMismatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster, task, params, ctx := twoReplicaTask(t, 9611, "ec-idx-vacuum") + httpClient := framework.NewHTTPClient() + const volumeID = uint32(9611) + + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{10, 11, 12}) + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{10, 11, 12}) + + // Delete the first needle so vacuum packs 11/12 to new offsets on A only. + deletedFID := framework.NewFileID(volumeID, 10, cookieFor(10)) + for _, url := range []string{cluster.VolumeAdminURL(0), cluster.VolumeAdminURL(1)} { + req, err := http.NewRequest(http.MethodDelete, url+"/"+deletedFID, nil) + require.NoError(t, err) + resp := framework.DoRequest(t, httpClient, req) + _ = framework.ReadAllAndClose(t, resp) + require.Equal(t, http.StatusAccepted, resp.StatusCode) + } + + connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0)) + defer connA.Close() + compactVolumeOnce(t, clientA, volumeID) + + offA := needleOffset(t, filepath.Join(cluster.VolumeDiskDir(0, 0), "ec-idx-vacuum_9611.idx"), 11) + offB := needleOffset(t, filepath.Join(cluster.VolumeDiskDir(1, 0), "ec-idx-vacuum_9611.idx"), 11) + require.NotEqual(t, offA, offB, "vacuum should rewrite offsets on one replica") + + require.NoError(t, task.Execute(ctx, params)) +} + +func TestEcEncodeAbortsWhenIdxRpcFails(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster, task, _, ctx := twoReplicaTask(t, 9612, "ec-idx-rpc") + httpClient := framework.NewHTTPClient() + const volumeID = uint32(9612) + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(0), volumeID, []uint64{1}) + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(1), volumeID, []uint64{1}) + + require.NoError(t, task.markReplicasReadonly(ctx)) + + locs := []wdclient.Location{ + {Url: serverAddress(cluster, 0)}, + {Url: "127.0.0.1:1"}, + } + _, err := volume_replica.SelectSourceIfIndexesMatch(task.grpcDialOption, needle.VolumeId(volumeID), "ec-idx-rpc", locs, "", &bytes.Buffer{}) + require.Error(t, err) +} + +func TestEcEncodeSingleReplicaAfterReadonly(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(9613) + framework.AllocateVolume(t, grpcClient, volumeID, "ec-idx-one") + httpClient := framework.NewHTTPClient() + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(), volumeID, []uint64{7, 8}) + + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + task := NewErasureCodingTask("ec-idx-one", cluster.VolumeServerAddress(), volumeID, "ec-idx-one", dialOption) + params := singleReplicaParams(cluster.VolumeServerAddress(), volumeID, "ec-idx-one", t.TempDir()) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + require.NoError(t, task.markReplicasReadonly(ctx)) + require.NoError(t, task.syncAndSelectSourceReplica()) + require.NoError(t, task.Execute(ctx, params)) +} + +func TestHttpWriteAfterReadonlyIsNotCreated(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + cluster := framework.StartVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(9614) + framework.AllocateVolume(t, grpcClient, volumeID, "") + httpClient := framework.NewHTTPClient() + uploadNeedles(t, httpClient, cluster.VolumeAdminURL(), volumeID, []uint64{1}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true}) + require.NoError(t, err) + + fid := framework.NewFileID(volumeID, 99, cookieFor(99)) + resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("must-not-land")) + _ = framework.ReadAllAndClose(t, resp) + require.NotEqual(t, http.StatusCreated, resp.StatusCode) + require.NotEqual(t, http.StatusNoContent, resp.StatusCode) + + idxPath := filepath.Join(cluster.BaseDir(), "volume", fmt.Sprintf("%d.idx", volumeID)) + require.False(t, idxHasNeedle(t, idxPath, 99)) +} + +func twoReplicaTask(t *testing.T, volumeID uint32, collection string) (*framework.MultiVolumeCluster, *ErasureCodingTask, *worker_pb.TaskParams, context.Context) { + t.Helper() + cluster := framework.StartMultiVolumeCluster(t, matrix.P1(), 2) + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + addrA := serverAddress(cluster, 0) + addrB := serverAddress(cluster, 1) + + connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0)) + t.Cleanup(func() { connA.Close() }) + connB, clientB := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1)) + t.Cleanup(func() { connB.Close() }) + framework.AllocateVolume(t, clientA, volumeID, collection) + framework.AllocateVolume(t, clientB, volumeID, collection) + + dataShards := int(erasure_coding.DataShardsCount) + totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount) + task := NewErasureCodingTask("ec-idx-gate", addrA, volumeID, collection, dialOption) + params := &worker_pb.TaskParams{ + VolumeId: volumeID, + Collection: collection, + Sources: []*worker_pb.TaskSource{ + {Node: addrA, VolumeId: volumeID}, + {Node: addrB, VolumeId: volumeID}, + }, + Targets: []*worker_pb.TaskTarget{ + {Node: addrA, ShardIds: shardRange(0, dataShards)}, + {Node: addrB, ShardIds: shardRange(dataShards, totalShards)}, + }, + TaskParams: &worker_pb.TaskParams_ErasureCodingParams{ + ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{ + DataShards: erasure_coding.DataShardsCount, + ParityShards: erasure_coding.ParityShardsCount, + WorkingDir: t.TempDir(), + }, + }, + } + task.sources = params.Sources + task.targets = params.Targets + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + t.Cleanup(cancel) + return cluster, task, params, ctx +} + +func singleReplicaParams(node string, volumeID uint32, collection, workDir string) *worker_pb.TaskParams { + totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount) + return &worker_pb.TaskParams{ + VolumeId: volumeID, + Collection: collection, + Sources: []*worker_pb.TaskSource{{Node: node, VolumeId: volumeID}}, + Targets: []*worker_pb.TaskTarget{{Node: node, ShardIds: shardRange(0, totalShards)}}, + TaskParams: &worker_pb.TaskParams_ErasureCodingParams{ + ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{ + DataShards: erasure_coding.DataShardsCount, + ParityShards: erasure_coding.ParityShardsCount, + WorkingDir: workDir, + }, + }, + } +} + +func cookieFor(id uint64) uint32 { + return 0xEC100000 + uint32(id) +} + +func uploadNeedles(t *testing.T, client *http.Client, volumeURL string, volumeID uint32, ids []uint64) { + t.Helper() + for _, id := range ids { + fid := framework.NewFileID(volumeID, id, cookieFor(id)) + resp := framework.UploadBytes(t, client, volumeURL, fid, []byte(fmt.Sprintf("payload-%d", id))) + _ = framework.ReadAllAndClose(t, resp) + require.Equal(t, http.StatusCreated, resp.StatusCode) + } +} + +func idxHasNeedle(t *testing.T, idxPath string, id uint64) bool { + t.Helper() + f, err := os.Open(idxPath) + require.NoError(t, err) + defer f.Close() + want := types.NeedleId(id) + found := false + err = idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error { + if key == want && !offset.IsZero() && !size.IsDeleted() { + found = true + } + return nil + }) + require.NoError(t, err) + return found +} + +func needleOffset(t *testing.T, idxPath string, id uint64) types.Offset { + t.Helper() + f, err := os.Open(idxPath) + require.NoError(t, err) + defer f.Close() + want := types.NeedleId(id) + var off types.Offset + var seen bool + err = idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error { + if key == want && !offset.IsZero() && !size.IsDeleted() { + off = offset + seen = true + } + return nil + }) + require.NoError(t, err) + require.True(t, seen, "needle %d missing in %s", id, idxPath) + return off +} diff --git a/weed/worker/tasks/erasure_coding/ec_task_julor_layout_e2e_test.go b/weed/worker/tasks/erasure_coding/ec_task_julor_layout_e2e_test.go index 5961f2e67..600aac92e 100644 --- a/weed/worker/tasks/erasure_coding/ec_task_julor_layout_e2e_test.go +++ b/weed/worker/tasks/erasure_coding/ec_task_julor_layout_e2e_test.go @@ -2,10 +2,7 @@ package erasure_coding import ( "context" - "fmt" "net/http" - "os" - "path/filepath" "testing" "time" @@ -19,20 +16,10 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// Reproduces the issue-9478 volume-13 layout end to end on a multi-server, -// multi-disk cluster and asserts the cluster converges to exactly one valid EC -// layout. Before the encode the volume is in the stuck state: a real .dat -// source, a 0-byte stub replica left by an interrupted encode, and partial -// stale EC shards from that attempt. After the worker runs (sweep stubs -> -// clear stale shards -> distribute -> mount -> verify -> delete originals): -// -// - every assigned shard is present on its target disk, with .ecx/.vif -// alongside it (the EC volume's info file survives even when shards land on -// the source's own disk); -// - no regular .dat/.idx remains anywhere — the source is deleted after -// verify, the stub was swept before distribute; -// - the stale shards that aren't part of the new assignment are gone. -func TestEcEncodeJulorLayoutConverges(t *testing.T) { +// A real .dat source plus a 0-byte stub replica left by an interrupted encode +// and partial stale EC shards: empty live-idx vs non-empty must refuse encode +// (no union, no skip). +func TestEcEncodeJulorLayoutRefuses(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } @@ -122,52 +109,8 @@ func TestEcEncodeJulorLayoutConverges(t *testing.T) { }, } - require.NoError(t, task.Execute(ctx, params)) - - base := fmt.Sprintf("%s_%d", collection, volumeID) - - // Shards are distributed per-server; the disk within a server is auto-selected - // when DiskId is 0, and on a multi-disk server shards can spread to a sibling - // disk while the .ecx/.vif index lives on one of them (the cross-disk EC - // layout). So assert per server (aggregate across its disks): each server - // holds exactly its assigned shards, its .ecx/.vif survive on at least one - // disk (the info file is not stripped, even on the source's own disk), and no - // regular .dat/.idx remains. Stale shards 11-13 mounted on the source must be - // gone — they are not in S0's assignment. - serverShards := map[int][]uint32{ - 0: shardRange(0, 5), // S0 (source): 0..4 - 1: shardRange(5, 10), // S1 (stub): 5..9 - 2: shardRange(10, 14), // S2 (target): 10..13 - } - allShards := map[uint32]bool{} - for server, want := range serverShards { - found := map[uint32]bool{} - hasEcx, hasVif := false, false - for d := 0; d < disksPerServer; d++ { - dir := cluster.VolumeDiskDir(server, d) - requireAbsent(t, dir, base+".dat") - requireAbsent(t, dir, base+".idx") - for id := 0; id < erasure_coding.TotalShardsCount; id++ { - if ecFileExists(dir, fmt.Sprintf("%s.ec%02d", base, id)) { - found[uint32(id)] = true - allShards[uint32(id)] = true - } - } - hasEcx = hasEcx || ecFileExists(dir, base+".ecx") - hasVif = hasVif || ecFileExists(dir, base+".vif") - } - gotIDs := make([]uint32, 0, len(found)) - for id := range found { - gotIDs = append(gotIDs, id) - } - require.ElementsMatch(t, want, gotIDs, "server %d should hold exactly its assigned shards", server) - require.True(t, hasEcx, "server %d must keep its .ecx", server) - require.True(t, hasVif, "server %d must keep its .vif (not stripped by the original-volume delete)", server) - } - require.Len(t, allShards, erasure_coding.TotalShardsCount, "the full shard set must be present across the cluster") -} - -func ecFileExists(dir, name string) bool { - _, err := os.Stat(filepath.Join(dir, name)) - return err == nil + err = task.Execute(ctx, params) + require.Error(t, err, "empty live-idx vs non-empty must refuse encode") + require.Contains(t, err.Error(), "no union") + require.NotEqual(t, addr(stubServer), task.server, "after gate t.server is not a stub URL") } diff --git a/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go b/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go index e2cc65b1b..38b08487c 100644 --- a/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go +++ b/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go @@ -1,31 +1,34 @@ package erasure_coding -import "testing" +import ( + "testing" -func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) { - a := replicaFileCount{url: "full-01:8080", fileCount: 21} - b := replicaFileCount{url: "full-02:8081", fileCount: 21} + "github.com/seaweedfs/seaweedfs/weed/wdclient" +) + +func TestLateBindSourceToWorkerHostPrefersWorkerHost(t *testing.T) { + a := wdclient.Location{Url: "full-01:8080"} + b := wdclient.Location{Url: "full-02:8081"} worker := "full-02:9334" // Sources [A,B] — topology-first is A; worker on B must still copy B. - if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url { - t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url) + if got := lateBindSourceToWorkerHost(a.Url, worker, []wdclient.Location{a, b}); got != b.Url { + t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.Url) } // Invert Sources — still B. - if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url { - t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url) + if got := lateBindSourceToWorkerHost(b.Url, worker, []wdclient.Location{b, a}); got != b.Url { + t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.Url) } } -func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) { - a := replicaFileCount{url: "full-01:8080", fileCount: 40} - b := replicaFileCount{url: "full-02:8081", fileCount: 21} - worker := "full-02:9334" +func TestLateBindSourceToWorkerHostNoMatchKeepsCurrent(t *testing.T) { + a := wdclient.Location{Url: "full-01:8080"} + b := wdclient.Location{Url: "full-02:8081"} - if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url { - t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url) + if got := lateBindSourceToWorkerHost(a.Url, "other-host:9334", []wdclient.Location{a, b}); got != a.Url { + t.Fatalf("no worker-host replica: got %s want %s", got, a.Url) } - if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url { - t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url) + if got := lateBindSourceToWorkerHost(a.Url, "", []wdclient.Location{a, b}); got != a.Url { + t.Fatalf("empty worker address: got %s want %s", got, a.Url) } }