seaweedfs/weed/admin/dash/collect_collection_stats_test.go

package dash

import (
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)

// TestCollectCollectionStatsECUnevenShards verifies that EC shards spread
// across multiple disks are aggregated correctly, including the case where
// the last data shard is smaller than the rest.
func TestCollectCollectionStatsECUnevenShards(t *testing.T) {
	// Standard 10+4 EC layout. Shards 0..9 are data, 10..13 are parity.
	// Data shards 0..8 are 1000 bytes; data shard 9 is 500 bytes (uneven tail).
	// Parity shards 10..13 are 1000 bytes each.
	//
	// Physical (raw) = 9*1000 + 500 + 4*1000 = 13500
	// Logical (data only) = 9*1000 + 500 = 9500

	nodeA := &master_pb.DataNodeInfo{
		DiskInfos: map[string]*master_pb.DiskInfo{
			"disk1": {
				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
					{
						Id:         42,
						Collection: "bucket-a",
						// Shards 0..6 held here.
						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
					},
				},
			},
		},
	}
	nodeB := &master_pb.DataNodeInfo{
		DiskInfos: map[string]*master_pb.DiskInfo{
			"disk1": {
				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
					{
						Id:         42,
						Collection: "bucket-a",
						// Shards 7..13 held here. Shard 9 (data) is the short tail.
						EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
						ShardSizes:  []int64{1000, 1000, 500, 1000, 1000, 1000, 1000},
					},
				},
			},
		},
	}

	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
					},
				},
			},
		},
	}

	stats := collectCollectionStats(topo)

	got, ok := stats["bucket-a"]
	if !ok {
		t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
	}

	const wantPhysical int64 = 13500
	const wantLogical int64 = 9500

	if got.PhysicalSize != wantPhysical {
		t.Errorf("PhysicalSize: got %d, want %d", got.PhysicalSize, wantPhysical)
	}
	if got.LogicalSize != wantLogical {
		t.Errorf("LogicalSize: got %d, want %d", got.LogicalSize, wantLogical)
	}
}

// TestCollectCollectionStatsECEmptyCollection verifies that EC shards with
// an empty collection name are bucketed under "default".
func TestCollectCollectionStatsECEmptyCollection(t *testing.T) {
	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							{
								DiskInfos: map[string]*master_pb.DiskInfo{
									"disk1": {
										EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
											{
												Id:          1,
												Collection:  "",
												EcIndexBits: (1 << 0) | (1 << 10),
												ShardSizes:  []int64{2000, 2000},
											},
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	stats := collectCollectionStats(topo)

	got, ok := stats["default"]
	if !ok {
		t.Fatalf("expected collection default in stats, got: %v", stats)
	}
	if got.PhysicalSize != 4000 {
		t.Errorf("PhysicalSize: got %d, want 4000", got.PhysicalSize)
	}
	// Only shard 0 is a data shard; shard 10 is parity.
	if got.LogicalSize != 2000 {
		t.Errorf("LogicalSize: got %d, want 2000", got.LogicalSize)
	}
}

// TestCollectCollectionStatsECFileAndDeleteCountAggregation verifies that
// FileCount for an EC volume is deduped across nodes (every shard holder has
// an identical .ecx, so the total entry count is taken once) while
// DeleteCount is summed (each needle delete tombstones exactly one node's
// .ecx, so the true delete total is the sum of every holder's local count).
func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) {
	// Volume id=7 reported by three nodes. Every node reports file_count=100
	// (same .ecx). Local delete counts: 5 + 3 + 2 = 10 deletes total.
	// Expected live object count = 100 - 10 = 90.
	makeNode := func(bits uint32, sizes []int64, deleteCount uint64) *master_pb.DataNodeInfo {
		return &master_pb.DataNodeInfo{
			DiskInfos: map[string]*master_pb.DiskInfo{
				"disk1": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{
							Id:          7,
							Collection:  "bucket-a",
							EcIndexBits: bits,
							ShardSizes:  sizes,
							FileCount:   100,
							DeleteCount: deleteCount,
						},
					},
				},
			},
		}
	}

	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3), []int64{1000, 1000, 1000, 1000}, 5),
							makeNode((1<<4)|(1<<5)|(1<<6)|(1<<7), []int64{1000, 1000, 1000, 1000}, 3),
							makeNode((1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1000, 1000, 1000, 1000, 1000, 1000}, 2),
						},
					},
				},
			},
		},
	}

	stats := collectCollectionStats(topo)

	got, ok := stats["bucket-a"]
	if !ok {
		t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
	}
	if got.FileCount != 90 {
		t.Errorf("FileCount: got %d, want 90 (100 total - 10 deletes summed)", got.FileCount)
	}
	// Sanity: 14 shards × 1000 bytes physical, 10 data shards logical.
	if got.PhysicalSize != 14000 {
		t.Errorf("PhysicalSize: got %d, want 14000", got.PhysicalSize)
	}
	if got.LogicalSize != 10000 {
		t.Errorf("LogicalSize: got %d, want 10000", got.LogicalSize)
	}
}

// TestCollectCollectionStatsECDeleteCountAboveFileCount verifies that an EC
// volume whose summed delete_count exceeds its file_count adds nothing to the
// collection's object count, while other volumes in the collection still do.
func TestCollectCollectionStatsECDeleteCountAboveFileCount(t *testing.T) {
	shard := func(vid uint32, bits uint32, fileCount, deleteCount uint64) *master_pb.VolumeEcShardInformationMessage {
		return &master_pb.VolumeEcShardInformationMessage{
			Id:          vid,
			Collection:  "bucket-c",
			EcIndexBits: bits,
			ShardSizes:  []int64{1},
			FileCount:   fileCount,
			DeleteCount: deleteCount,
		}
	}
	node := func(shards ...*master_pb.VolumeEcShardInformationMessage) *master_pb.DataNodeInfo {
		return &master_pb.DataNodeInfo{
			DiskInfos: map[string]*master_pb.DiskInfo{"disk1": {EcShardInfos: shards}},
		}
	}

	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							node(shard(21, 1<<0, 100, 60), shard(22, 1<<0, 50, 10)),
							node(shard(21, 1<<1, 100, 60)),
						},
					},
				},
			},
		},
	}

	got := collectCollectionStats(topo)["bucket-c"]
	if got.FileCount != 40 {
		t.Errorf("FileCount: got %d, want 40 (volume 21 skipped, volume 22 has 50-10)", got.FileCount)
	}
}

// TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count
// is taken as the max across reporting nodes rather than the first-seen
// value. A node that has not yet finished loading .ecx reports file_count=0,
// which previously poisoned the aggregate and rendered buckets backed by EC
// volumes as "0 objects".
func TestCollectCollectionStatsECFileCountMaxDedupe(t *testing.T) {
	makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo {
		return &master_pb.DataNodeInfo{
			DiskInfos: map[string]*master_pb.DiskInfo{
				"disk1": {
					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
						{
							Id:          11,
							Collection:  "bucket-b",
							EcIndexBits: bits,
							ShardSizes:  sizes,
							FileCount:   fileCount,
						},
					},
				},
			},
		}
	}

	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						// First-reporting node has a stale fileCount of 0,
						// second node reports the authoritative 6.
						DataNodeInfos: []*master_pb.DataNodeInfo{
							makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0),
							makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6),
						},
					},
				},
			},
		},
	}

	stats := collectCollectionStats(topo)
	got, ok := stats["bucket-b"]
	if !ok {
		t.Fatalf("expected collection bucket-b in stats, got: %v", stats)
	}
	if got.FileCount != 6 {
		t.Errorf("FileCount: got %d, want 6 (max across reporters)", got.FileCount)
	}
}

// TestTotalCollectionFileCount verifies the cluster-wide chunk count counts a
// chunk once per replicated volume and per EC volume, and nets out deletes.
func TestTotalCollectionFileCount(t *testing.T) {
	// Volume 1 has replication 001 (two copies), so both nodes report the same
	// 101 chunks with 10 deleted: 91 live chunks in total, not 182 and not the
	// 90 that halving each report would give.
	replica := func() *master_pb.DiskInfo {
		return &master_pb.DiskInfo{
			VolumeInfos: []*master_pb.VolumeInformationMessage{
				{
					Id:               1,
					Collection:       "bucket-a",
					ReplicaPlacement: 1,
					FileCount:        101,
					DeleteCount:      10,
				},
			},
		}
	}
	// EC volume 2 has its 20 chunks reported by every shard holder.
	ecShards := func(bits uint32, sizes []int64) *master_pb.DiskInfo {
		return &master_pb.DiskInfo{
			EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
				{
					Id:          2,
					Collection:  "bucket-b",
					EcIndexBits: bits,
					ShardSizes:  sizes,
					FileCount:   20,
				},
			},
		}
	}

	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							{DiskInfos: map[string]*master_pb.DiskInfo{
								"disk1": replica(),
								"disk2": ecShards((1<<0)|(1<<1), []int64{1, 1}),
							}},
							{DiskInfos: map[string]*master_pb.DiskInfo{
								"disk1": replica(),
								"disk2": ecShards((1<<2)|(1<<3), []int64{1, 1}),
							}},
						},
					},
				},
			},
		},
	}

	if got := totalCollectionFileCount(topo); got != 111 {
		t.Errorf("totalCollectionFileCount: got %d, want 111 (91 replicated + 20 EC)", got)
	}
}

// TestTotalCollectionFileCountUnderReplicated verifies a volume whose second
// replica has not reported yet still contributes its full live count.
func TestTotalCollectionFileCountUnderReplicated(t *testing.T) {
	topo := &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							{DiskInfos: map[string]*master_pb.DiskInfo{
								"disk1": {
									VolumeInfos: []*master_pb.VolumeInformationMessage{
										{
											Id:               1,
											Collection:       "bucket-a",
											ReplicaPlacement: 1,
											FileCount:        50,
										},
									},
								},
							}},
						},
					},
				},
			},
		},
	}

	if got := totalCollectionFileCount(topo); got != 50 {
		t.Errorf("totalCollectionFileCount: got %d, want 50 (one replica reporting, not halved)", got)
	}
}