mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-12 01:07:35 +02:00
storage: unify needle-map counters across index loaders and warn on copy-count drift (#11701)
* storage: count index-load metrics only for live-key removals The in-memory and leveldb offset loaders counted a deletion for every tombstone row replayed, including re-deletes of already-deleted keys (runtime logDelete only counts a live removal), and added the returned old size to the byte counters even when it was a tombstone sentinel — uint64(-1) wraps DeletionByteCounter. The leveldb offset loader also counted every index row as a file and its bytes unconditionally. Gate the counters the same way the runtime paths do so a reload reports the same numbers the runtime counters hold. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * storage: derive index metrics from latest-row state in the metric loader The leveldb metric loader walked the index in reverse and incremented FileCounter for every row (inflating FileCount with tombstones) and DeletionCounter for every superseded row plus every deleted-key row (double counting both). Per key, the runtime deleted count is exactly (valid rows) - (1 when the latest row is live), so count live-latest rows via the same bloom filter and derive both deletion counters; this matches the runtime logPut/logDelete counters exactly instead of drifting with tombstone history. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * server: warn instead of failing volume copies on counter drift checkCopyCounts compared the source's in-memory counters against the target's fresh-load counters, but counters computed by different loaders drifted apart for identical .idx data, so a good copy was rejected (issue #11678). The file-size checks already gate integrity; log the mismatch instead of failing the copy. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * storage: assert MaxFileKey parity in the metric loader test MaxFileKey derives from row order alone, so unlike the bloom-filtered deletion counters it must always match the runtime counters exactly. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
5f1a742726
commit
4bf5e92b64
6 files changed
+99
-37
No files matched your search
@@ -259,7 +259,13 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
|
||||
if !shouldValidateCopyCounts {
|
||||
return nil
|
||||
}
|
||||
return checkCopyCounts(sourceVolumeStatusAfterCopy, targetVolume.FileCount(), targetVolume.DeletedCount())
|
||||
// Counter drift (bloom-filter estimates, rows appended between the
|
||||
// status snapshots) is not proof of a bad copy; the byte sizes
|
||||
// already match. Warn instead of rejecting the copy.
|
||||
if countErr := checkCopyCounts(sourceVolumeStatusAfterCopy, targetVolume.FileCount(), targetVolume.DeletedCount()); countErr != nil {
|
||||
glog.Warningf("copied volume %d counts differ from source in-memory counters (counter drift): %v", req.VolumeId, countErr)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to mount or validate volume %d: %w", req.VolumeId, err)
|
||||
@@ -302,7 +308,7 @@ func (vs *VolumeServer) doCopyFileWithThrottler(client volume_server_pb.VolumeSe
|
||||
|
||||
}
|
||||
|
||||
// checkCopyFiles verifies the copied file sizes. Record counts are checked
|
||||
// checkCopyFiles verifies the copied file sizes. Record counts are compared
|
||||
// after the target volume is mounted, when the target needle map is available.
|
||||
func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse, hasRemoteDatFile bool, idxFileName, datFileName string) error {
|
||||
stat, err := os.Stat(idxFileName)
|
||||
|
||||
Reference in new issue
Block a user