admin: summarize the EC delete_count accounting warning per refresh (#12359)

* admin: summarize the EC delete_count accounting warning per refresh

collectCollectionStats runs on every admin collection, bucket and
topology refresh, and it logged a Warning for each EC volume whose
delete_count summed across shard holders exceeds its file_count. That
sum is an expected accounting artefact (a tombstone can be recorded on
more than one holder), it recurs identically on every pass, and on one
cluster with hundreds of such volumes it grew the admin log to 20 GB,
more than 99% of it this one line.

Log one V(1) summary per pass with the affected volume count and keep
the per-volume detail at V(2). The object count is still skipped for
those volumes.

* admin: test that an EC volume with deletes above file_count is skipped

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-03 06:31:15 -08:00
1 parent e5bc658eb7
commit 222cfa8e2e
2 files changed
+51 -2

No files matched your search

+10 -2
View File
@@ -2309,17 +2309,25 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
// Fold EC per-volume counts into the collection totals. fileCount is
// deduped via max across every node reporting shards for the volume;
// deleteCount is summed across the same nodes.
// deleteCount is summed across the same nodes, and a tombstone recorded
// on more than one shard holder can push that sum past fileCount. This
// is an expected accounting artefact that recurs on every refresh, so
// it is summarized once per pass rather than logged per volume.
skipped := 0
for vid, agg := range ecVolumeAgg {
data := collectionMap[agg.collection]
if agg.fileCount >= agg.deleteCount {
data.FileCount += int64(agg.fileCount - agg.deleteCount)
} else {
glog.Warningf("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count",
skipped++
glog.V(2).Infof("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count",
vid, agg.collection, agg.deleteCount, agg.fileCount)
}
collectionMap[agg.collection] = data
}
if skipped > 0 {
glog.V(1).Infof("%d ec volumes have summed delete_count above file_count; skipping object count for them", skipped)
}
return collectionMap
}
@@ -185,6 +185,47 @@ func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) {
}
}
// TestCollectCollectionStatsECDeleteCountAboveFileCount verifies that an EC
// volume whose summed delete_count exceeds its file_count adds nothing to the
// collection's object count, while other volumes in the collection still do.
func TestCollectCollectionStatsECDeleteCountAboveFileCount(t *testing.T) {
shard := func(vid uint32, bits uint32, fileCount, deleteCount uint64) *master_pb.VolumeEcShardInformationMessage {
return &master_pb.VolumeEcShardInformationMessage{
Id: vid,
Collection: "bucket-c",
EcIndexBits: bits,
ShardSizes: []int64{1},
FileCount: fileCount,
DeleteCount: deleteCount,
}
}
node := func(shards ...*master_pb.VolumeEcShardInformationMessage) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{"disk1": {EcShardInfos: shards}},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
node(shard(21, 1<<0, 100, 60), shard(22, 1<<0, 50, 10)),
node(shard(21, 1<<1, 100, 60)),
},
},
},
},
},
}
got := collectCollectionStats(topo)["bucket-c"]
if got.FileCount != 40 {
t.Errorf("FileCount: got %d, want 40 (volume 21 skipped, volume 22 has 50-10)", got.FileCount)
}
}
// TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count
// is taken as the max across reporting nodes rather than the first-seen
// value. A node that has not yet finished loading .ecx reports file_count=0,