diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index ded05c301..79189eb30 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -549,9 +549,127 @@ func (vcd *volumeCheckDisk) checkBoth(source, target *VolumeReplica, bidi bool) return sourceHasChanges, targetHasChanges, errors.Join(errs...) } + // When nothing was repaired — typically because resurrection is gated off + // since both replicas have been vacuumed (compaction revision > 0), which is + // the normal state of any production cluster — doVolumeCheckDisk only logged + // "cannot prove they are missing writes vs vacuumed deletes" and stopped. + // That dead-end leaves a diverged replica with no actionable path, so classify + // the divergence and print the exact repair. This is report-only: it changes + // no data and does not bypass the resurrection safety gate. + if !targetHasChanges && !sourceHasChanges { + sourceOnly, targetOnly := vcd.liveDivergence(sourceDB, targetDB) + if sourceOnly > 0 || targetOnly > 0 { + vcd.reportDivergenceVerdict(source, target, sourceOnly, targetOnly, + sourceRevision, targetRevision, + vcd.resurrectMissingNeedles && bidi, vcd.resurrectMissingNeedles, vcd.resurrectMissingNeedles) + } + } + return sourceHasChanges, targetHasChanges, nil } +// liveDivergence counts live (non-deleted) needles present on a's index but +// entirely absent from b's, and the reverse. It is used to classify a diverged +// replica pair: one-sided (the lagging replica holds no unique live data, so a +// re-copy of the complete side converges it — subject to the deletion caveat +// reportDivergenceVerdict states, since the absent needles may be valid +// deletions on a vacuumed replica) versus two-sided (true split-brain — do +// not auto-repair). Tombstones are excluded, so vacuum asymmetry (a compacted +// replica that has dropped deleted entries) does not create a false +// difference. +func (vcd *volumeCheckDisk) liveDivergence(a, b *needle_map.MemDb) (aOnly, bOnly int) { + a.DescendingVisit(func(v needle_map.NeedleValue) error { + if v.Size.IsDeleted() { + return nil + } + if _, found := b.Get(v.Key); !found { + aOnly++ + } + return nil + }) + b.DescendingVisit(func(v needle_map.NeedleValue) error { + if v.Size.IsDeleted() { + return nil + } + if _, found := a.Get(v.Key); !found { + bOnly++ + } + return nil + }) + return +} + +// reportDivergenceVerdict prints the actionable outcome for a diverged replica +// pair that check.disk could not repair in place. revSrc/revTgt are the +// replicas' compaction revisions (0 = never vacuumed; >0 = at least one +// vacuum dropped deleted entries from the index); srcRevKnown/tgtRevKnown +// report whether each revision was actually read (target is read even in +// unidirectional mode; source only under -bidirectional). resurrectFlag is +// the command line's -resurrectMissingNeedles. +// +// The verdict is tombstone-aware: when the lagging replica has been vacuumed, +// a needle that is live on the complete side but absent on the lagging side is +// ambiguous — either a missing write, or a valid deletion whose tombstone the +// lagging side's vacuum already dropped. A whole-volume re-copy converges the +// divergence either way, but in the latter case it resurrects deleted data, so +// the volume.copy command is only emitted when the lagging side is proven +// never-vacuumed; otherwise the verdict points at a non-destructive +// needle-level repair instead. +func (vcd *volumeCheckDisk) reportDivergenceVerdict(source, target *VolumeReplica, sourceOnly, targetOnly int, revSrc, revTgt uint32, srcRevKnown, tgtRevKnown, resurrectFlag bool) { + // Raw string form, NOT String(): ServerAddress.String() drops the custom + // gRPC port suffix, and volume.copy's dialer accepts the host:port.grpcPort + // form (see checkDialable in weed/operation/volume_move). + srcAddr := string(pb.NewServerAddressFromDataNode(source.location.dataNode)) + tgtAddr := string(pb.NewServerAddressFromDataNode(target.location.dataNode)) + vid := source.info.Id + switch { + case sourceOnly > 0 && targetOnly == 0: + // target is lagging: it holds no unique live data. + safe := deletionCaveat(resurrectFlag, tgtRevKnown, revTgt == 0) + if safe { + vcd.write("volume %d: ONE-SIDED divergence — %s is missing %d live needle(s) that exist on %s; %s holds no unique live data. Safe repair (whole-volume re-copy, complete -> lagging; verify-before-destroy is enforced): volume.copy -source %s -target %s -volumeId %d", + vid, tgtAddr, sourceOnly, srcAddr, tgtAddr, srcAddr, tgtAddr, vid) + } else { + vcd.write("volume %d: ONE-SIDED divergence — %s is missing %d live needle(s) that exist on %s; %s holds no unique live data. Do NOT re-copy the whole volume: the absent needles may be valid deletions already vacuumed away on the lagging side, which a re-copy would resurrect. Restore only the confirmed-missing needles (volume.fsck -collection -volumeId %d -findMissingChunksInFiler, then needle-level repair), or re-copy only after accepting that risk.", + vid, tgtAddr, sourceOnly, srcAddr, tgtAddr, vid) + } + case targetOnly > 0 && sourceOnly == 0: + // source is lagging: it holds no unique live data. + safe := deletionCaveat(resurrectFlag, srcRevKnown, revSrc == 0) + if safe { + vcd.write("volume %d: ONE-SIDED divergence — %s is missing %d live needle(s) that exist on %s; %s holds no unique live data. Safe repair (whole-volume re-copy, complete -> lagging; verify-before-destroy is enforced): volume.copy -source %s -target %s -volumeId %d", + vid, srcAddr, targetOnly, tgtAddr, srcAddr, tgtAddr, srcAddr, vid) + } else { + vcd.write("volume %d: ONE-SIDED divergence — %s is missing %d live needle(s) that exist on %s; %s holds no unique live data. Do NOT re-copy the whole volume: the absent needles may be valid deletions already vacuumed away on the lagging side, which a re-copy would resurrect. Restore only the confirmed-missing needles (volume.fsck -collection -volumeId %d -findMissingChunksInFiler, then needle-level repair), or re-copy only after accepting that risk.", + vid, srcAddr, targetOnly, tgtAddr, srcAddr, vid) + } + case sourceOnly > 0 && targetOnly > 0: + if resurrectFlag && srcRevKnown && tgtRevKnown && revSrc == 0 && revTgt == 0 { + // Both replicas proven never-vacuumed: the mutually missing + // needles are provably missing writes on both sides (the same + // proof the resurrection gate uses), not split-brain. The two + // doVolumeCheckDisk passes above already queued them for + // in-place resurrection but this was a simulation run, so + // nothing was applied. + vcd.write("volume %d: TWO-SIDED divergence, both replicas never vacuumed — %s is missing %d live needle(s) that exist on %s AND %s is missing %d that exist on %s. These are mutually missed writes, not split-brain: re-run this command with -apply to resurrect them in place (both directions).", + vid, tgtAddr, sourceOnly, srcAddr, srcAddr, targetOnly, tgtAddr) + } else { + vcd.write("volume %d: TWO-SIDED (split-brain) divergence — %s has %d unique live needle(s) AND %s has %d. Do NOT auto-repair: each side may hold data the other lacks. Confirm orphans with volume.fsck -collection -volumeId %d -findMissingChunksInFiler before re-copying the complete replica, or restore the missing needles manually.", + vid, srcAddr, sourceOnly, tgtAddr, targetOnly, vid) + } + } +} + +// deletionCaveat reports whether a one-sided divergence's complete -> lagging +// re-copy is safe: the lagging side is proven never-vacuumed (compaction +// revision 0) and that proof was actually taken under the +// -resurrectMissingNeedles flag, so its absent live needles are missing +// writes by the same proof the resurrection gate uses — not vacuumed +// deletions. +func deletionCaveat(resurrectFlag, laggingRevKnown, laggingNeverVacuumed bool) bool { + return resurrectFlag && laggingRevKnown && laggingNeverVacuumed +} + func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.MemDb, source, target *VolumeReplica, resurrectAbsent bool, targetRevision uint32) (hasChanges bool, err error) { // find missing keys diff --git a/weed/shell/command_volume_check_disk_test.go b/weed/shell/command_volume_check_disk_test.go index 2615b0b26..243f03a6c 100644 --- a/weed/shell/command_volume_check_disk_test.go +++ b/weed/shell/command_volume_check_disk_test.go @@ -2,6 +2,7 @@ package shell import ( "bytes" + "strings" "testing" "time" @@ -12,6 +13,201 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +// TestLiveDivergenceClassifiesOneSidedVsTwoSided verifies the divergence +// classifier used to turn check.disk's dead-end "cannot prove" no-op into an +// actionable verdict: one-sided (safe to re-copy) vs two-sided (split-brain). +func TestLiveDivergenceClassifiesOneSidedVsTwoSided(t *testing.T) { + // one-sided: source has an extra live needle; target has none unique. + src, tgt := needle_map.NewMemDb(), needle_map.NewMemDb() + defer src.Close() + defer tgt.Close() + if err := src.Set(types.NeedleId(1001), types.ToOffset(8), types.Size(123)); err != nil { + t.Fatalf("seed: %v", err) + } + if err := tgt.Set(types.NeedleId(1002), types.ToOffset(8), types.Size(45)); err != nil { + t.Fatalf("seed: %v", err) + } + // a shared live needle (should NOT count as divergence either way) + if err := src.Set(types.NeedleId(2000), types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed shared: %v", err) + } + if err := tgt.Set(types.NeedleId(2000), types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed shared: %v", err) + } + vcd := &volumeCheckDisk{writer: &bytes.Buffer{}, now: time.Now()} + sOnly, tOnly := vcd.liveDivergence(src, tgt) + if sOnly != 1 || tOnly != 1 { + t.Fatalf("one-sided: got sOnly=%d tOnly=%d, want 1/1 (1001 only on src, 1002 only on tgt)", sOnly, tOnly) + } + + // strict subset: target has no unique needles -> tOnly must be 0. + subset, full := needle_map.NewMemDb(), needle_map.NewMemDb() + defer subset.Close() + defer full.Close() + if err := subset.Set(types.NeedleId(2000), types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed: %v", err) + } + for _, id := range []types.NeedleId{2000, 2001, 2002} { + if err := full.Set(id, types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed: %v", err) + } + } + aOnly, bOnly := vcd.liveDivergence(full, subset) // full->subset + if aOnly != 2 || bOnly != 0 { + t.Fatalf("subset: full is superset; want aOnly=2 (2001,2002) bOnly=0, got %d/%d", aOnly, bOnly) + } + + // tombstones must be excluded: a deleted-only difference is not divergence. + delSrc, delTgt := needle_map.NewMemDb(), needle_map.NewMemDb() + defer delSrc.Close() + defer delTgt.Close() + if err := delSrc.Set(types.NeedleId(3000), types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed: %v", err) + } + if err := delTgt.Set(types.NeedleId(3000), types.ToOffset(8), types.Size(9)); err != nil { + t.Fatalf("seed: %v", err) + } + // delete 3000 on the source as a real tombstone (negative size), leave it + // live on target. liveDivergence must skip the tombstone on the source and + // must not count 3000 on the target either, because Get still finds the + // tombstone entry — so the expected answer is 0/0, not 0/1. + if err := delSrc.Set(types.NeedleId(3000), types.ToOffset(8), types.Size(-9)); err != nil { + t.Fatalf("tombstone: %v", err) + } + x, y := vcd.liveDivergence(delSrc, delTgt) + if x != 0 || y != 0 { + t.Fatalf("tombstone: deleted needle on src should not count; want src-only=0, tgt-only=0, got %d/%d", x, y) + } +} + +// TestReportDivergenceVerdictEmitsCopyCommand checks that a one-sided verdict +// prints the exact volume.copy command (complete -> lagging, dialable address), +// that the command never uses the logical node Id, that a vacuumed (or +// unproven) lagging replica gets the no-re-copy warning instead of a copy +// command, that a proven never-vacuumed lagging replica gets the safe copy +// command — including in unidirectional mode where only the target's revision +// is known — and that a two-sided verdict warns instead of auto-repairing. +func TestReportDivergenceVerdictEmitsCopyCommand(t *testing.T) { + // Id is a logical node identifier (NOT dialable); Address is the real + // ip:port the copy command must use. + src := &VolumeReplica{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "node-1", Address: "10.0.0.1:8081"}}, info: &master_pb.VolumeInformationMessage{Id: 42}} + tgt := &VolumeReplica{location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "node-2", Address: "10.0.0.2:8083"}}, info: &master_pb.VolumeInformationMessage{Id: 42}} + + // Case A: source has 5 unique live, target has none -> target is lagging. + // Revisions were not read (both known flags false), so no copy command. + var one bytes.Buffer + vcd := &volumeCheckDisk{writer: &one, now: time.Now()} + vcd.reportDivergenceVerdict(src, tgt, 5, 0, 0, 0, false, false, false) + got := one.String() + if !strings.Contains(got, "ONE-SIDED") { + t.Fatalf("expected ONE-SIDED verdict, got: %s", got) + } + if strings.Contains(got, "volume.copy") { + t.Fatalf("unproven lagging replica must not get a volume.copy command, got: %s", got) + } + if !strings.Contains(got, "Do NOT re-copy the whole volume") { + t.Fatalf("expected no-re-copy warning for unproven lagging replica, got: %s", got) + } + if !strings.Contains(got, "10.0.0.1:8081") || !strings.Contains(got, "10.0.0.2:8083") { + t.Fatalf("expected dialable addresses in verdict, got: %s", got) + } + if strings.Contains(got, "node-1") || strings.Contains(got, "node-2") { + t.Fatalf("verdict must use dialable Address, not logical node Id, got: %s", got) + } + if !strings.Contains(got, "5") { + t.Fatalf("expected missing-needle count 5, got: %s", got) + } + + // Case B: source is lagging (target complete). The safe command must go + // complete -> lagging, i.e. -source -target . + var safeRev bytes.Buffer + vcdSafeRev := &volumeCheckDisk{writer: &safeRev, now: time.Now()} + vcdSafeRev.reportDivergenceVerdict(src, tgt, 0, 5, 0, 0, true, false, true) + gotRev := safeRev.String() + if !strings.Contains(gotRev, "volume.copy -source 10.0.0.2:8083 -target 10.0.0.1:8081 -volumeId 42") { + t.Fatalf("expected copy complete(10.0.0.2:8083)->lagging(10.0.0.1:8081), got: %s", gotRev) + } + if strings.Contains(gotRev, "volume.copy -source 10.0.0.1:8081 -target 10.0.0.2:8083") { + t.Fatalf("reversed direction: must not copy lagging source over complete target, got: %s", gotRev) + } + + // Case C: target-lagging, unidirectional run (-resurrectMissingNeedles, + // not -bidirectional): sourceRevision was never read (srcRevKnown=false) + // but targetRevision was (tgtRevKnown=true) and is 0. The safe copy + // command must still be emitted — the false-warning regression. + var uni bytes.Buffer + vcdUni := &volumeCheckDisk{writer: &uni, now: time.Now()} + vcdUni.reportDivergenceVerdict(src, tgt, 5, 0, 0, 0, false, true, true) + gotUni := uni.String() + if !strings.Contains(gotUni, "ONE-SIDED") { + t.Fatalf("expected ONE-SIDED verdict, got: %s", gotUni) + } + if !strings.Contains(gotUni, "volume.copy -source 10.0.0.1:8081 -target 10.0.0.2:8083 -volumeId 42") { + t.Fatalf("unidirectional run with proven never-vacuumed target must emit the safe copy command, got: %s", gotUni) + } + if strings.Contains(gotUni, "Do NOT re-copy the whole volume") { + t.Fatalf("false warning: proven never-vacuumed lagging target must not get the no-re-copy warning, got: %s", gotUni) + } + + // Case D: target-lagging but the lagging side was vacuumed (rev 17) -> + // no copy command, even though the proof was taken under the flag. + var vac bytes.Buffer + vcdVac := &volumeCheckDisk{writer: &vac, now: time.Now()} + vcdVac.reportDivergenceVerdict(src, tgt, 5, 0, 17, 17, true, true, true) + gotVac := vac.String() + if strings.Contains(gotVac, "volume.copy") { + t.Fatalf("vacuumed lagging replica must not get a volume.copy command, got: %s", gotVac) + } + if !strings.Contains(gotVac, "Do NOT re-copy the whole volume") { + t.Fatalf("expected no-re-copy warning for vacuumed lagging replica, got: %s", gotVac) + } + + // Case E: two-sided, unproven (revisions not read) -> split-brain warning. + var two bytes.Buffer + vcd2 := &volumeCheckDisk{writer: &two, now: time.Now()} + vcd2.reportDivergenceVerdict(src, tgt, 3, 4, 0, 0, false, false, false) + got2 := two.String() + if !strings.Contains(got2, "TWO-SIDED") || !strings.Contains(got2, "split-brain") { + t.Fatalf("expected TWO-SIDED split-brain warning, got: %s", got2) + } + if strings.Contains(got2, "volume.copy") { + t.Fatalf("two-sided must not emit an auto volume.copy command, got: %s", got2) + } + + // Case F: two-sided with BOTH replicas proven never-vacuumed under the + // resurrection flag (bidirectional simulation): mutually missing writes, + // not split-brain — the in-place -apply repair must be recommended and + // the split-brain warning must not appear. + var both bytes.Buffer + vcdBoth := &volumeCheckDisk{writer: &both, now: time.Now()} + vcdBoth.reportDivergenceVerdict(src, tgt, 3, 4, 0, 0, true, true, true) + gotBoth := both.String() + if !strings.Contains(gotBoth, "TWO-SIDED") { + t.Fatalf("expected TWO-SIDED verdict, got: %s", gotBoth) + } + if strings.Contains(gotBoth, "Do NOT auto-repair") { + t.Fatalf("proven never-vacuumed both sides must not get the split-brain warning, got: %s", gotBoth) + } + if !strings.Contains(gotBoth, "mutually missed writes") || !strings.Contains(gotBoth, "-apply") { + t.Fatalf("expected mutually-missed-writes verdict recommending -apply, got: %s", gotBoth) + } + if strings.Contains(gotBoth, "volume.copy") { + t.Fatalf("two-sided must not emit an auto volume.copy command, got: %s", gotBoth) + } + + // Case G: custom gRPC ports must survive into the emitted addresses + // (ServerAddress.String() would drop the .grpcPort suffix). + srcGrpc := &VolumeReplica{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "node-1", Address: "10.0.0.1:8081", GrpcPort: 18081}}, info: &master_pb.VolumeInformationMessage{Id: 42}} + tgtGrpc := &VolumeReplica{location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "node-2", Address: "10.0.0.2:8083", GrpcPort: 18083}}, info: &master_pb.VolumeInformationMessage{Id: 42}} + var grpc bytes.Buffer + vcdGrpc := &volumeCheckDisk{writer: &grpc, now: time.Now()} + vcdGrpc.reportDivergenceVerdict(srcGrpc, tgtGrpc, 5, 0, 0, 0, false, true, true) + gotGrpc := grpc.String() + if !strings.Contains(gotGrpc, "10.0.0.1:8081.18081") || !strings.Contains(gotGrpc, "10.0.0.2:8083.18083") { + t.Fatalf("custom gRPC port must be preserved in emitted addresses, got: %s", gotGrpc) + } +} + // TestDoVolumeCheckDiskDoesNotResurrectAbsentNeedle verifies that a needle // present-and-live on the source but entirely absent on the target is NOT // pushed back by default. Such an absence is indistinguishable from a needle