mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
shell: volume.check.disk skips replicas whose live counts match (#12743)
* shell: volume.check.disk skips replicas whose live counts match FileCount counts every write and DeleteCount every delete or overwrite until a vacuum drops them, so a replica vacuumed while its peer was not keeps a different FileCount forever and was re-checked on every run although both hold the same live needles. Only treat counts as differing when the live counts (FileCount - DeleteCount) differ too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: volume.check.disk skips equal live counts only across compaction revisions Equal live counts at the same compaction revision mean a missed write and a missed delete, not a vacuum-state difference, so keep checking those pairs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: volume.check.disk real-time recheck ignores heartbeat revisions VolumeStatus counts are newer than the heartbeat's compaction revisions, so a vacuum in between could make a pair at the same revision look vacuum-skewed. Apply the old count rules there. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: volume.check.disk skips on live counts only when no repair could follow Equal live counts across compaction revisions can still hide a missed write and a missed delete. Keep checking such pairs when -syncDeleted could propagate the delete, or when -resurrectMissingNeedles could copy a needle into a never-vacuumed replica. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e037a9c1ce
commit
9f413ff2c8
2 files changed
+180
-33
No files matched your search
@@ -377,34 +377,50 @@ func (vcd *volumeCheckDisk) getVolumeStatusFileCount(vid uint32, dn *master_pb.D
|
||||
}
|
||||
|
||||
// eqVolumeFileCount compares the real-time file counts of two volume replicas
|
||||
// by making sequential gRPC calls to their volume servers.
|
||||
//
|
||||
// Returns:
|
||||
// - bool: true if file counts match
|
||||
// - bool: true if deleted file counts match
|
||||
// - error: any error from volume server communication
|
||||
//
|
||||
// Error Handling: Errors from getVolumeStatusFileCount are wrapped with context
|
||||
// (volume ID and server) and propagated up. Uses fmt.Errorf with %w to maintain
|
||||
// error chain for errors.Is() and errors.As().
|
||||
func (vcd *volumeCheckDisk) eqVolumeFileCount(a, b *VolumeReplica) (bool, bool, error) {
|
||||
// by making sequential gRPC calls to their volume servers, and reports whether
|
||||
// they agree as countsDiffer judges them.
|
||||
func (vcd *volumeCheckDisk) eqVolumeFileCount(a, b *VolumeReplica) (bool, error) {
|
||||
fileCountA, fileDeletedCountA, errA := vcd.getVolumeStatusFileCount(a.info.Id, a.location.dataNode)
|
||||
if errA != nil {
|
||||
return false, false, fmt.Errorf("getting volume %d status from %s: %w", a.info.Id, a.location.dataNode.Id, errA)
|
||||
return false, fmt.Errorf("getting volume %d status from %s: %w", a.info.Id, a.location.dataNode.Id, errA)
|
||||
}
|
||||
|
||||
fileCountB, fileDeletedCountB, errB := vcd.getVolumeStatusFileCount(b.info.Id, b.location.dataNode)
|
||||
if errB != nil {
|
||||
return false, false, fmt.Errorf("getting volume %d status from %s: %w", b.info.Id, b.location.dataNode.Id, errB)
|
||||
return false, fmt.Errorf("getting volume %d status from %s: %w", b.info.Id, b.location.dataNode.Id, errB)
|
||||
}
|
||||
|
||||
return fileCountA == fileCountB, fileDeletedCountA == fileDeletedCountB, nil
|
||||
// VolumeStatus reports no compaction revision to pair with these counts.
|
||||
return !vcd.countsDiffer(fileCountA, fileDeletedCountA, fileCountB, fileDeletedCountB, false), nil
|
||||
}
|
||||
|
||||
// countsDiffer reports whether two replicas' needle counters suggest a needle
|
||||
// is missing on one side. FileCount counts every write and DeleteCount every
|
||||
// delete or overwrite until a vacuum drops them; with liveCountsSuffice, equal
|
||||
// live counts are taken as a vacuum-state difference.
|
||||
func (vcd *volumeCheckDisk) countsDiffer(fileCountA, deleteCountA, fileCountB, deleteCountB uint64, liveCountsSuffice bool) bool {
|
||||
if liveCountsSuffice && int64(fileCountA)-int64(deleteCountA) == int64(fileCountB)-int64(deleteCountB) {
|
||||
return false
|
||||
}
|
||||
return fileCountA != fileCountB || (vcd.syncDeletions && deleteCountA != deleteCountB)
|
||||
}
|
||||
|
||||
// liveCountsSuffice reports whether equal live counts are enough to skip two
|
||||
// replicas at different compaction revisions. Equal live counts can also hide
|
||||
// a missed write plus a missed delete, so this holds only when no repair this
|
||||
// run could make depends on them: deletes are not synced and neither replica
|
||||
// can take a resurrected needle.
|
||||
func (vcd *volumeCheckDisk) liveCountsSuffice(a, b *VolumeReplica) bool {
|
||||
if vcd.syncDeletions || a.info.CompactRevision == b.info.CompactRevision {
|
||||
return false
|
||||
}
|
||||
return !vcd.resurrectMissingNeedles || (a.info.CompactRevision > 0 && b.info.CompactRevision > 0)
|
||||
}
|
||||
|
||||
// shouldSkipVolume determines whether two volume replicas should skip synchronization.
|
||||
//
|
||||
// Logic:
|
||||
// 1. If file counts and delete counts match (when syncDeletions enabled), skip sync
|
||||
// 1. If countsDiffer finds no missing needle, skip sync
|
||||
// 2. If counts differ AND both volumes were modified recently (>= pulseTimeAtSecond),
|
||||
// they may still be actively receiving writes, so we return true to skip sync and
|
||||
// avoid false positives
|
||||
@@ -424,29 +440,21 @@ func (vcd *volumeCheckDisk) shouldSkipVolume(a, b *VolumeReplica) (bool, error)
|
||||
}
|
||||
|
||||
pulseTimeAtSecond := vcd.now.Add(-constants.VolumePulsePeriod * 2).Unix()
|
||||
doSyncDeletedCount := false
|
||||
if vcd.syncDeletions && a.info.DeleteCount != b.info.DeleteCount {
|
||||
doSyncDeletedCount = true
|
||||
}
|
||||
if (a.info.FileCount != b.info.FileCount) || doSyncDeletedCount {
|
||||
if vcd.countsDiffer(a.info.FileCount, a.info.DeleteCount, b.info.FileCount, b.info.DeleteCount, vcd.liveCountsSuffice(a, b)) {
|
||||
// Do synchronization of volumes, if the modification time was before the last pulsation time
|
||||
if a.info.ModifiedAtSecond < pulseTimeAtSecond || b.info.ModifiedAtSecond < pulseTimeAtSecond {
|
||||
return false, nil
|
||||
}
|
||||
eqFileCount, eqDeletedFileCount, err := vcd.eqVolumeFileCount(a, b)
|
||||
eqFileCount, err := vcd.eqVolumeFileCount(a, b)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("comparing volume %d file counts on %s and %s: %w",
|
||||
a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id, err)
|
||||
}
|
||||
if eqFileCount {
|
||||
if doSyncDeletedCount && !eqDeletedFileCount {
|
||||
return false, nil
|
||||
}
|
||||
vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s",
|
||||
a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
|
||||
} else {
|
||||
if !eqFileCount {
|
||||
return false, nil
|
||||
}
|
||||
vcd.writeVerbose("skipping active volumes %d with the same file counts on %s and %s",
|
||||
a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -367,6 +367,7 @@ type shouldSkipVolume struct {
|
||||
b VolumeReplica
|
||||
pulseTimeAtSecond int64
|
||||
syncDeletions bool
|
||||
resurrect bool
|
||||
shouldSkipVolume bool
|
||||
}
|
||||
|
||||
@@ -573,7 +574,7 @@ func TestShouldSkipVolume(t *testing.T) {
|
||||
},
|
||||
// Edge case: Both volumes modified AFTER pulse cutoff time
|
||||
// When ModifiedAtSecond >= pulseTimeAtSecond for both volumes with same counts,
|
||||
// the condition (a.info.FileCount != b.info.FileCount) is false, so we skip
|
||||
// countsDiffer is false, so we skip
|
||||
// without calling eqVolumeFileCount
|
||||
{
|
||||
name: "both volumes modified after pulse cutoff with same file counts should be skipped",
|
||||
@@ -591,15 +592,153 @@ func TestShouldSkipVolume(t *testing.T) {
|
||||
syncDeletions: true,
|
||||
shouldSkipVolume: true, // Same counts = skip without calling eqVolumeFileCount
|
||||
},
|
||||
{
|
||||
name: "replicas differing only in vacuum state should be skipped",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 900,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: false,
|
||||
shouldSkipVolume: true,
|
||||
},
|
||||
{
|
||||
name: "replicas at different revisions with equal live counts should be checked with syncDeletions enabled",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 900,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: true,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "vacuumed replica with a different live count should not be skipped",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 899,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: false,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "missed write and missed delete at the same revision should not be skipped",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1001,
|
||||
DeleteCount: 101,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: true,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "missed write and missed delete at the same revision should not be skipped without syncDeletions",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 2,
|
||||
DeleteCount: 1,
|
||||
CompactRevision: 3,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 3,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: false,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "vacuumed replica that missed a write and a delete should be checked with syncDeletions enabled",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 3,
|
||||
DeleteCount: 2,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
syncDeletions: true,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "never-vacuumed replica should be checked when resurrection is enabled",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 900,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
resurrect: true,
|
||||
shouldSkipVolume: false,
|
||||
},
|
||||
{
|
||||
name: "vacuumed replicas at different revisions should be skipped when resurrection is enabled",
|
||||
a: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 1000,
|
||||
DeleteCount: 100,
|
||||
CompactRevision: 1,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
b: VolumeReplica{nil, &master_pb.VolumeInformationMessage{
|
||||
FileCount: 900,
|
||||
DeleteCount: 0,
|
||||
CompactRevision: 2,
|
||||
ModifiedAtSecond: 1696583300},
|
||||
},
|
||||
pulseTimeAtSecond: 1696583400,
|
||||
resurrect: true,
|
||||
shouldSkipVolume: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
vcd := &volumeCheckDisk{
|
||||
writer: &buf,
|
||||
now: time.Unix(tt.pulseTimeAtSecond, 0),
|
||||
verbose: false, // reduce noise in tests
|
||||
syncDeletions: tt.syncDeletions,
|
||||
writer: &buf,
|
||||
now: time.Unix(tt.pulseTimeAtSecond, 0),
|
||||
verbose: false, // reduce noise in tests
|
||||
syncDeletions: tt.syncDeletions,
|
||||
resurrectMissingNeedles: tt.resurrect,
|
||||
}
|
||||
result, err := vcd.shouldSkipVolume(&tt.a, &tt.b)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user