From 222cfa8e2e0a441a91dc6dff04782dced44968f4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 3 Nov 2026 06:31:15 -0800 Subject: [PATCH] admin: summarize the EC delete_count accounting warning per refresh (#12359) * admin: summarize the EC delete_count accounting warning per refresh collectCollectionStats runs on every admin collection, bucket and topology refresh, and it logged a Warning for each EC volume whose delete_count summed across shard holders exceeds its file_count. That sum is an expected accounting artefact (a tombstone can be recorded on more than one holder), it recurs identically on every pass, and on one cluster with hundreds of such volumes it grew the admin log to 20 GB, more than 99% of it this one line. Log one V(1) summary per pass with the affected volume count and keep the per-volume detail at V(2). The object count is still skipped for those volumes. * admin: test that an EC volume with deletes above file_count is skipped Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/admin/dash/admin_server.go | 12 +++++- .../dash/collect_collection_stats_test.go | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index fd9abe2e0d..27958dec6d 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -2309,17 +2309,25 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col // Fold EC per-volume counts into the collection totals. fileCount is // deduped via max across every node reporting shards for the volume; - // deleteCount is summed across the same nodes. + // deleteCount is summed across the same nodes, and a tombstone recorded + // on more than one shard holder can push that sum past fileCount. This + // is an expected accounting artefact that recurs on every refresh, so + // it is summarized once per pass rather than logged per volume. + skipped := 0 for vid, agg := range ecVolumeAgg { data := collectionMap[agg.collection] if agg.fileCount >= agg.deleteCount { data.FileCount += int64(agg.fileCount - agg.deleteCount) } else { - glog.Warningf("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count", + skipped++ + glog.V(2).Infof("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count", vid, agg.collection, agg.deleteCount, agg.fileCount) } collectionMap[agg.collection] = data } + if skipped > 0 { + glog.V(1).Infof("%d ec volumes have summed delete_count above file_count; skipping object count for them", skipped) + } return collectionMap } diff --git a/weed/admin/dash/collect_collection_stats_test.go b/weed/admin/dash/collect_collection_stats_test.go index 5451ba6369..2d26610e3e 100644 --- a/weed/admin/dash/collect_collection_stats_test.go +++ b/weed/admin/dash/collect_collection_stats_test.go @@ -185,6 +185,47 @@ func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) { } } +// TestCollectCollectionStatsECDeleteCountAboveFileCount verifies that an EC +// volume whose summed delete_count exceeds its file_count adds nothing to the +// collection's object count, while other volumes in the collection still do. +func TestCollectCollectionStatsECDeleteCountAboveFileCount(t *testing.T) { + shard := func(vid uint32, bits uint32, fileCount, deleteCount uint64) *master_pb.VolumeEcShardInformationMessage { + return &master_pb.VolumeEcShardInformationMessage{ + Id: vid, + Collection: "bucket-c", + EcIndexBits: bits, + ShardSizes: []int64{1}, + FileCount: fileCount, + DeleteCount: deleteCount, + } + } + node := func(shards ...*master_pb.VolumeEcShardInformationMessage) *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{"disk1": {EcShardInfos: shards}}, + } + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{ + node(shard(21, 1<<0, 100, 60), shard(22, 1<<0, 50, 10)), + node(shard(21, 1<<1, 100, 60)), + }, + }, + }, + }, + }, + } + + got := collectCollectionStats(topo)["bucket-c"] + if got.FileCount != 40 { + t.Errorf("FileCount: got %d, want 40 (volume 21 skipped, volume 22 has 50-10)", got.FileCount) + } +} + // TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count // is taken as the max across reporting nodes rather than the first-seen // value. A node that has not yet finished loading .ecx reports file_count=0,