mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-15 10:38:35 +02:00
Chris Lu
ef77df6141
* admin: include EC volumes in bucket size reporting The Object Store buckets page computed per-collection size by iterating only regular volumes, so once a bucket's data was EC-encoded it silently disappeared from the reported size — breaking usage-based billing. Walk EcShardInfos alongside VolumeInfos in collectCollectionStats: add raw shard bytes to PhysicalSize, and the parity-stripped value (shardBytes * DataShardsCount / TotalShardsCount) to LogicalSize, matching the normalization used by `weed shell` cluster.status. * admin: derive EC logical size from shard bitmap, not constants Use ShardsInfoFromVolumeEcShardInformationMessage + MinusParityShards to sum actual data-shard bytes instead of scaling raw bytes by the DataShardsCount/TotalShardsCount ratio. Keeps the data/parity split encapsulated in the erasure_coding package and is exact when shard sizes differ (e.g. last shard). * admin: regression test for EC shard size aggregation Cover the uneven-tail-shard case (data shard 9 < 1000 bytes) and the empty-collection-name path to pin PhysicalSize/LogicalSize behavior for collectCollectionStats against future changes.
125 lines
3.4 KiB
Go
125 lines
3.4 KiB
Go
package dash
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
)
|
|
|
|
// TestCollectCollectionStatsECUnevenShards verifies that EC shards spread
|
|
// across multiple disks are aggregated correctly, including the case where
|
|
// the last data shard is smaller than the rest.
|
|
func TestCollectCollectionStatsECUnevenShards(t *testing.T) {
|
|
// Standard 10+4 EC layout. Shards 0..9 are data, 10..13 are parity.
|
|
// Data shards 0..8 are 1000 bytes; data shard 9 is 500 bytes (uneven tail).
|
|
// Parity shards 10..13 are 1000 bytes each.
|
|
//
|
|
// Physical (raw) = 9*1000 + 500 + 4*1000 = 13500
|
|
// Logical (data only) = 9*1000 + 500 = 9500
|
|
|
|
nodeA := &master_pb.DataNodeInfo{
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"disk1": {
|
|
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
|
{
|
|
Id: 42,
|
|
Collection: "bucket-a",
|
|
// Shards 0..6 held here.
|
|
EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
|
|
ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
nodeB := &master_pb.DataNodeInfo{
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"disk1": {
|
|
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
|
{
|
|
Id: 42,
|
|
Collection: "bucket-a",
|
|
// Shards 7..13 held here. Shard 9 (data) is the short tail.
|
|
EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
|
|
ShardSizes: []int64{1000, 1000, 500, 1000, 1000, 1000, 1000},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
topo := &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
stats := collectCollectionStats(topo)
|
|
|
|
got, ok := stats["bucket-a"]
|
|
if !ok {
|
|
t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
|
|
}
|
|
|
|
const wantPhysical int64 = 13500
|
|
const wantLogical int64 = 9500
|
|
|
|
if got.PhysicalSize != wantPhysical {
|
|
t.Errorf("PhysicalSize: got %d, want %d", got.PhysicalSize, wantPhysical)
|
|
}
|
|
if got.LogicalSize != wantLogical {
|
|
t.Errorf("LogicalSize: got %d, want %d", got.LogicalSize, wantLogical)
|
|
}
|
|
}
|
|
|
|
// TestCollectCollectionStatsECEmptyCollection verifies that EC shards with
|
|
// an empty collection name are bucketed under "default".
|
|
func TestCollectCollectionStatsECEmptyCollection(t *testing.T) {
|
|
topo := &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"disk1": {
|
|
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
|
{
|
|
Id: 1,
|
|
Collection: "",
|
|
EcIndexBits: (1 << 0) | (1 << 10),
|
|
ShardSizes: []int64{2000, 2000},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
stats := collectCollectionStats(topo)
|
|
|
|
got, ok := stats["default"]
|
|
if !ok {
|
|
t.Fatalf("expected collection default in stats, got: %v", stats)
|
|
}
|
|
if got.PhysicalSize != 4000 {
|
|
t.Errorf("PhysicalSize: got %d, want 4000", got.PhysicalSize)
|
|
}
|
|
// Only shard 0 is a data shard; shard 10 is parity.
|
|
if got.LogicalSize != 2000 {
|
|
t.Errorf("LogicalSize: got %d, want 2000", got.LogicalSize)
|
|
}
|
|
}
|