From 9f413ff2c8a176fe1308c2e5669294848b63de62 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2026 14:10:47 -0800 Subject: [PATCH] shell: volume.check.disk skips replicas whose live counts match (#12743) * shell: volume.check.disk skips replicas whose live counts match FileCount counts every write and DeleteCount every delete or overwrite until a vacuum drops them, so a replica vacuumed while its peer was not keeps a different FileCount forever and was re-checked on every run although both hold the same live needles. Only treat counts as differing when the live counts (FileCount - DeleteCount) differ too. Co-Authored-By: Claude Opus 5.5 * shell: volume.check.disk skips equal live counts only across compaction revisions Equal live counts at the same compaction revision mean a missed write and a missed delete, not a vacuum-state difference, so keep checking those pairs. Co-Authored-By: Claude Opus 5.5 * shell: volume.check.disk real-time recheck ignores heartbeat revisions VolumeStatus counts are newer than the heartbeat's compaction revisions, so a vacuum in between could make a pair at the same revision look vacuum-skewed. Apply the old count rules there. Co-Authored-By: Claude Opus 5.5 * shell: volume.check.disk skips on live counts only when no repair could follow Equal live counts across compaction revisions can still hide a missed write and a missed delete. Keep checking such pairs when -syncDeleted could propagate the delete, or when -resurrectMissingNeedles could copy a needle into a never-vacuumed replica. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/shell/command_volume_check_disk.go | 64 ++++---- weed/shell/command_volume_check_disk_test.go | 149 ++++++++++++++++++- 2 files changed, 180 insertions(+), 33 deletions(-) diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index 01e45dc1ba..edd9416446 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -377,34 +377,50 @@ func (vcd *volumeCheckDisk) getVolumeStatusFileCount(vid uint32, dn *master_pb.D } // eqVolumeFileCount compares the real-time file counts of two volume replicas -// by making sequential gRPC calls to their volume servers. -// -// Returns: -// - bool: true if file counts match -// - bool: true if deleted file counts match -// - error: any error from volume server communication -// -// Error Handling: Errors from getVolumeStatusFileCount are wrapped with context -// (volume ID and server) and propagated up. Uses fmt.Errorf with %w to maintain -// error chain for errors.Is() and errors.As(). -func (vcd *volumeCheckDisk) eqVolumeFileCount(a, b *VolumeReplica) (bool, bool, error) { +// by making sequential gRPC calls to their volume servers, and reports whether +// they agree as countsDiffer judges them. +func (vcd *volumeCheckDisk) eqVolumeFileCount(a, b *VolumeReplica) (bool, error) { fileCountA, fileDeletedCountA, errA := vcd.getVolumeStatusFileCount(a.info.Id, a.location.dataNode) if errA != nil { - return false, false, fmt.Errorf("getting volume %d status from %s: %w", a.info.Id, a.location.dataNode.Id, errA) + return false, fmt.Errorf("getting volume %d status from %s: %w", a.info.Id, a.location.dataNode.Id, errA) } fileCountB, fileDeletedCountB, errB := vcd.getVolumeStatusFileCount(b.info.Id, b.location.dataNode) if errB != nil { - return false, false, fmt.Errorf("getting volume %d status from %s: %w", b.info.Id, b.location.dataNode.Id, errB) + return false, fmt.Errorf("getting volume %d status from %s: %w", b.info.Id, b.location.dataNode.Id, errB) } - return fileCountA == fileCountB, fileDeletedCountA == fileDeletedCountB, nil + // VolumeStatus reports no compaction revision to pair with these counts. + return !vcd.countsDiffer(fileCountA, fileDeletedCountA, fileCountB, fileDeletedCountB, false), nil +} + +// countsDiffer reports whether two replicas' needle counters suggest a needle +// is missing on one side. FileCount counts every write and DeleteCount every +// delete or overwrite until a vacuum drops them; with liveCountsSuffice, equal +// live counts are taken as a vacuum-state difference. +func (vcd *volumeCheckDisk) countsDiffer(fileCountA, deleteCountA, fileCountB, deleteCountB uint64, liveCountsSuffice bool) bool { + if liveCountsSuffice && int64(fileCountA)-int64(deleteCountA) == int64(fileCountB)-int64(deleteCountB) { + return false + } + return fileCountA != fileCountB || (vcd.syncDeletions && deleteCountA != deleteCountB) +} + +// liveCountsSuffice reports whether equal live counts are enough to skip two +// replicas at different compaction revisions. Equal live counts can also hide +// a missed write plus a missed delete, so this holds only when no repair this +// run could make depends on them: deletes are not synced and neither replica +// can take a resurrected needle. +func (vcd *volumeCheckDisk) liveCountsSuffice(a, b *VolumeReplica) bool { + if vcd.syncDeletions || a.info.CompactRevision == b.info.CompactRevision { + return false + } + return !vcd.resurrectMissingNeedles || (a.info.CompactRevision > 0 && b.info.CompactRevision > 0) } // shouldSkipVolume determines whether two volume replicas should skip synchronization. // // Logic: -// 1. If file counts and delete counts match (when syncDeletions enabled), skip sync +// 1. If countsDiffer finds no missing needle, skip sync // 2. If counts differ AND both volumes were modified recently (>= pulseTimeAtSecond), // they may still be actively receiving writes, so we return true to skip sync and // avoid false positives @@ -424,29 +440,21 @@ func (vcd *volumeCheckDisk) shouldSkipVolume(a, b *VolumeReplica) (bool, error) } pulseTimeAtSecond := vcd.now.Add(-constants.VolumePulsePeriod * 2).Unix() - doSyncDeletedCount := false - if vcd.syncDeletions && a.info.DeleteCount != b.info.DeleteCount { - doSyncDeletedCount = true - } - if (a.info.FileCount != b.info.FileCount) || doSyncDeletedCount { + if vcd.countsDiffer(a.info.FileCount, a.info.DeleteCount, b.info.FileCount, b.info.DeleteCount, vcd.liveCountsSuffice(a, b)) { // Do synchronization of volumes, if the modification time was before the last pulsation time if a.info.ModifiedAtSecond < pulseTimeAtSecond || b.info.ModifiedAtSecond < pulseTimeAtSecond { return false, nil } - eqFileCount, eqDeletedFileCount, err := vcd.eqVolumeFileCount(a, b) + eqFileCount, err := vcd.eqVolumeFileCount(a, b) if err != nil { return false, fmt.Errorf("comparing volume %d file counts on %s and %s: %w", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id, err) } - if eqFileCount { - if doSyncDeletedCount && !eqDeletedFileCount { - return false, nil - } - vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s", - a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id) - } else { + if !eqFileCount { return false, nil } + vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s", + a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id) } return true, nil } diff --git a/weed/shell/command_volume_check_disk_test.go b/weed/shell/command_volume_check_disk_test.go index 243f03a6c0..cfdb2472e4 100644 --- a/weed/shell/command_volume_check_disk_test.go +++ b/weed/shell/command_volume_check_disk_test.go @@ -367,6 +367,7 @@ type shouldSkipVolume struct { b VolumeReplica pulseTimeAtSecond int64 syncDeletions bool + resurrect bool shouldSkipVolume bool } @@ -573,7 +574,7 @@ func TestShouldSkipVolume(t *testing.T) { }, // Edge case: Both volumes modified AFTER pulse cutoff time // When ModifiedAtSecond >= pulseTimeAtSecond for both volumes with same counts, - // the condition (a.info.FileCount != b.info.FileCount) is false, so we skip + // countsDiffer is false, so we skip // without calling eqVolumeFileCount { name: "both volumes modified after pulse cutoff with same file counts should be skipped", @@ -591,15 +592,153 @@ func TestShouldSkipVolume(t *testing.T) { syncDeletions: true, shouldSkipVolume: true, // Same counts = skip without calling eqVolumeFileCount }, + { + name: "replicas differing only in vacuum state should be skipped", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 900, + DeleteCount: 0, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: false, + shouldSkipVolume: true, + }, + { + name: "replicas at different revisions with equal live counts should be checked with syncDeletions enabled", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 900, + DeleteCount: 0, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: true, + shouldSkipVolume: false, + }, + { + name: "vacuumed replica with a different live count should not be skipped", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 899, + DeleteCount: 0, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: false, + shouldSkipVolume: false, + }, + { + name: "missed write and missed delete at the same revision should not be skipped", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1001, + DeleteCount: 101, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: true, + shouldSkipVolume: false, + }, + { + name: "missed write and missed delete at the same revision should not be skipped without syncDeletions", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 2, + DeleteCount: 1, + CompactRevision: 3, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1, + DeleteCount: 0, + CompactRevision: 3, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: false, + shouldSkipVolume: false, + }, + { + name: "vacuumed replica that missed a write and a delete should be checked with syncDeletions enabled", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 3, + DeleteCount: 2, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1, + DeleteCount: 0, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + syncDeletions: true, + shouldSkipVolume: false, + }, + { + name: "never-vacuumed replica should be checked when resurrection is enabled", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 900, + DeleteCount: 0, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + resurrect: true, + shouldSkipVolume: false, + }, + { + name: "vacuumed replicas at different revisions should be skipped when resurrection is enabled", + a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 1000, + DeleteCount: 100, + CompactRevision: 1, + ModifiedAtSecond: 1696583300}, + }, + b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{ + FileCount: 900, + DeleteCount: 0, + CompactRevision: 2, + ModifiedAtSecond: 1696583300}, + }, + pulseTimeAtSecond: 1696583400, + resurrect: true, + shouldSkipVolume: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var buf bytes.Buffer vcd := &volumeCheckDisk{ - writer: &buf, - now: time.Unix(tt.pulseTimeAtSecond, 0), - verbose: false, // reduce noise in tests - syncDeletions: tt.syncDeletions, + writer: &buf, + now: time.Unix(tt.pulseTimeAtSecond, 0), + verbose: false, // reduce noise in tests + syncDeletions: tt.syncDeletions, + resurrectMissingNeedles: tt.resurrect, } result, err := vcd.shouldSkipVolume(&tt.a, &tt.b) if err != nil {