diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index fd9abe2e0d..27958dec6d 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -2309,17 +2309,25 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col // Fold EC per-volume counts into the collection totals. fileCount is // deduped via max across every node reporting shards for the volume; - // deleteCount is summed across the same nodes. + // deleteCount is summed across the same nodes, and a tombstone recorded + // on more than one shard holder can push that sum past fileCount. This + // is an expected accounting artefact that recurs on every refresh, so + // it is summarized once per pass rather than logged per volume. + skipped := 0 for vid, agg := range ecVolumeAgg { data := collectionMap[agg.collection] if agg.fileCount >= agg.deleteCount { data.FileCount += int64(agg.fileCount - agg.deleteCount) } else { - glog.Warningf("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count", + skipped++ + glog.V(2).Infof("ec volume %d in collection %q: summed delete_count=%d exceeds file_count=%d; skipping object count", vid, agg.collection, agg.deleteCount, agg.fileCount) } collectionMap[agg.collection] = data } + if skipped > 0 { + glog.V(1).Infof("%d ec volumes have summed delete_count above file_count; skipping object count for them", skipped) + } return collectionMap } diff --git a/weed/admin/dash/collect_collection_stats_test.go b/weed/admin/dash/collect_collection_stats_test.go index 5451ba6369..2d26610e3e 100644 --- a/weed/admin/dash/collect_collection_stats_test.go +++ b/weed/admin/dash/collect_collection_stats_test.go @@ -185,6 +185,47 @@ func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) { } } +// TestCollectCollectionStatsECDeleteCountAboveFileCount verifies that an EC +// volume whose summed delete_count exceeds its file_count adds nothing to the +// collection's object count, while other volumes in the collection still do. +func TestCollectCollectionStatsECDeleteCountAboveFileCount(t *testing.T) { + shard := func(vid uint32, bits uint32, fileCount, deleteCount uint64) *master_pb.VolumeEcShardInformationMessage { + return &master_pb.VolumeEcShardInformationMessage{ + Id: vid, + Collection: "bucket-c", + EcIndexBits: bits, + ShardSizes: []int64{1}, + FileCount: fileCount, + DeleteCount: deleteCount, + } + } + node := func(shards ...*master_pb.VolumeEcShardInformationMessage) *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{"disk1": {EcShardInfos: shards}}, + } + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{ + node(shard(21, 1<<0, 100, 60), shard(22, 1<<0, 50, 10)), + node(shard(21, 1<<1, 100, 60)), + }, + }, + }, + }, + }, + } + + got := collectCollectionStats(topo)["bucket-c"] + if got.FileCount != 40 { + t.Errorf("FileCount: got %d, want 40 (volume 21 skipped, volume 22 has 50-10)", got.FileCount) + } +} + // TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count // is taken as the max across reporting nodes rather than the first-seen // value. A node that has not yet finished loading .ecx reports file_count=0,