shell: volume.check.disk skips replicas whose live counts match (#12743)

* shell: volume.check.disk skips replicas whose live counts match

FileCount counts every write and DeleteCount every delete or overwrite
until a vacuum drops them, so a replica vacuumed while its peer was not
keeps a different FileCount forever and was re-checked on every run
although both hold the same live needles. Only treat counts as differing
when the live counts (FileCount - DeleteCount) differ too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* shell: volume.check.disk skips equal live counts only across compaction revisions

Equal live counts at the same compaction revision mean a missed write
and a missed delete, not a vacuum-state difference, so keep checking
those pairs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* shell: volume.check.disk real-time recheck ignores heartbeat revisions

VolumeStatus counts are newer than the heartbeat's compaction revisions,
so a vacuum in between could make a pair at the same revision look
vacuum-skewed. Apply the old count rules there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* shell: volume.check.disk skips on live counts only when no repair could follow

Equal live counts across compaction revisions can still hide a missed
write and a missed delete. Keep checking such pairs when -syncDeleted
could propagate the delete, or when -resurrectMissingNeedles could copy
a needle into a never-vacuumed replica.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-18 14:10:47 -08:00
1 parent e037a9c1ce
commit 9f413ff2c8
2 files changed
+180 -33

No files matched your search

+144 -5
View File
@@ -367,6 +367,7 @@ type shouldSkipVolume struct {
b VolumeReplica
pulseTimeAtSecond int64
syncDeletions bool
resurrect bool
shouldSkipVolume bool
}
@@ -573,7 +574,7 @@ func TestShouldSkipVolume(t *testing.T) {
},
// Edge case: Both volumes modified AFTER pulse cutoff time
// When ModifiedAtSecond >= pulseTimeAtSecond for both volumes with same counts,
// the condition (a.info.FileCount != b.info.FileCount) is false, so we skip
// countsDiffer is false, so we skip
// without calling eqVolumeFileCount
{
name: "both volumes modified after pulse cutoff with same file counts should be skipped",
@@ -591,15 +592,153 @@ func TestShouldSkipVolume(t *testing.T) {
syncDeletions: true,
shouldSkipVolume: true, // Same counts = skip without calling eqVolumeFileCount
},
{
name: "replicas differing only in vacuum state should be skipped",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 900,
DeleteCount: 0,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: false,
shouldSkipVolume: true,
},
{
name: "replicas at different revisions with equal live counts should be checked with syncDeletions enabled",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 900,
DeleteCount: 0,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: true,
shouldSkipVolume: false,
},
{
name: "vacuumed replica with a different live count should not be skipped",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 899,
DeleteCount: 0,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: false,
shouldSkipVolume: false,
},
{
name: "missed write and missed delete at the same revision should not be skipped",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1001,
DeleteCount: 101,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: true,
shouldSkipVolume: false,
},
{
name: "missed write and missed delete at the same revision should not be skipped without syncDeletions",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 2,
DeleteCount: 1,
CompactRevision: 3,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1,
DeleteCount: 0,
CompactRevision: 3,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: false,
shouldSkipVolume: false,
},
{
name: "vacuumed replica that missed a write and a delete should be checked with syncDeletions enabled",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 3,
DeleteCount: 2,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1,
DeleteCount: 0,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
syncDeletions: true,
shouldSkipVolume: false,
},
{
name: "never-vacuumed replica should be checked when resurrection is enabled",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 900,
DeleteCount: 0,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
resurrect: true,
shouldSkipVolume: false,
},
{
name: "vacuumed replicas at different revisions should be skipped when resurrection is enabled",
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 1000,
DeleteCount: 100,
CompactRevision: 1,
ModifiedAtSecond: 1696583300},
},
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
FileCount: 900,
DeleteCount: 0,
CompactRevision: 2,
ModifiedAtSecond: 1696583300},
},
pulseTimeAtSecond: 1696583400,
resurrect: true,
shouldSkipVolume: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var buf bytes.Buffer
vcd := &volumeCheckDisk{
writer: &buf,
now: time.Unix(tt.pulseTimeAtSecond, 0),
verbose: false, // reduce noise in tests
syncDeletions: tt.syncDeletions,
writer: &buf,
now: time.Unix(tt.pulseTimeAtSecond, 0),
verbose: false, // reduce noise in tests
syncDeletions: tt.syncDeletions,
resurrectMissingNeedles: tt.resurrect,
}
result, err := vcd.shouldSkipVolume(&tt.a, &tt.b)
if err != nil {