Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-11-02 10:17:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3: take bucket sizes from the master's summary (#10664)

Browse Source 
* pb: ask the master what each collection holds

Callers tracking usage were sent every volume in the cluster to add up
themselves, which is the master's largest single allocation.

* topology: summarise what each collection holds

One pass over the topology, allocating per collection rather than per volume.
Regular volumes count once each for logical totals and once per replica for
physical, taken from the lookup index, which is already keyed by volume and so
needs no set of seen ids. Ec shards are node-local so their sizes sum, while
the file and delete counts describe the volume and resolve once every holder
has been seen.

Replicas of one volume disagree while a write is landing or a heartbeat is
late. Walking a full listing took whichever replica the map iteration reached
first, so the answer moved between runs; this takes the largest, which is
stable and never reports usage below what some replica already holds.

* s3: take bucket sizes from the master's summary

The bucket size metrics pulled the whole volume list once a minute and added it
up, which cost the master 184.6MB of allocation and 17.8MB on the wire for six
numbers per collection.

  VolumeList over 550k volumes   184.6 MB allocated, 17.8 MB on the wire
  CollectionStatistics              176 bytes allocated, 47 bytes on the wire

The aggregation moves to the master with it, so the cases the removed tests
covered are now asserted against it directly.

* topology: count the replica holding the most live data

Quotas are enforced on size less deletions, and the replica with the biggest
raw size can be the one that has deleted the most. Counting it reported a
bucket smaller than it is and would leave one writable over its quota, which is
the opposite of what picking the largest was meant to guarantee.

* topology: cap a volume's deletions at what it holds

Live usage is read as a collection's size less its deletions, so a volume
reporting more deleted bytes than it has cancels live bytes belonging to other
volumes in the same bucket and reports it smaller than it is. Replica selection
already floored that volume's own live size at zero; the totals have to agree
with it.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-08-09 00:00:19 -07:00
1 parent 38db7e1493
commit 567052bfb6
9 files changed
+1054 -602
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

weed/s3api/bucket_size_metrics.go
+16 -115
View File
@@ -15,7 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
const (
@@ -46,12 +45,6 @@ func (c *CollectionInfo) LogicalSize() float64 {
return c.Size - c.DeletedByteCount
}
// volumeKey uniquely identifies a volume for deduplication
type volumeKey struct {
collection string
volumeId uint32
}
// startBucketSizeMetricsLoop periodically collects bucket size metrics and updates Prometheus gauges.
// Uses a distributed lock to ensure only one S3 instance collects metrics at a time.
// Should be called as a goroutine; stops when the provided context is cancelled.
@@ -185,18 +178,28 @@ func (s3a *S3ApiServer) collectCollectionInfoFromMaster(ctx context.Context) (ma
masterMap[string(master)] = master
}
// Connect to any available master and get volume list with topology
// Ask the master to summarise. Adding this up here instead would mean
// being sent every volume in the cluster once a minute.
collectionInfos := make(map[string]*CollectionInfo)
err := pb.WithOneOfGrpcMasterClients(false, masterMap, s3a.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{})
resp, err := client.CollectionStatistics(ctx, &master_pb.CollectionStatisticsRequest{})
if err != nil {
return fmt.Errorf("failed to get volume list: %w", err)
return fmt.Errorf("failed to get collection statistics: %w", err)
}
if resp == nil || resp.TopologyInfo == nil {
return fmt.Errorf("empty topology info from master")
if resp == nil {
return fmt.Errorf("empty collection statistics from master")
}
for _, c := range resp.Collections {
collectionInfos[c.Collection] = &CollectionInfo{
FileCount: float64(c.FileCount),
DeleteCount: float64(c.DeleteCount),
DeletedByteCount: float64(c.DeletedByteCount),
Size: float64(c.Size),
PhysicalSize: float64(c.PhysicalSize),
VolumeCount: int(c.VolumeCount),
}
}
collectCollectionInfoFromTopology(resp.TopologyInfo, collectionInfos)
return nil
})
if err != nil {
@@ -256,105 +259,3 @@ func (s3a *S3ApiServer) listBuckets(ctx context.Context) ([]*filer_pb.Entry, err
return buckets, err
}
// ecVolumeAgg accumulates per-volume EC counts across the shard holders.
// fileCount is volume-wide (every holder sees the same .ecx) so we take the
// max across reporters to avoid a slow node with a not-yet-loaded .ecx
// pinning the aggregate at 0. deleteCount is node-local to each .ecj
// deletion journal, so it's summed across reporters.
type ecVolumeAgg struct {
collection string
fileCount uint64
deleteCount uint64
}
// collectCollectionInfoFromTopology extracts collection info from topology.
// Deduplicates by volume ID to correctly handle missing replicas.
// Unlike dividing by copyCount (which would give wrong results if replicas are missing),
// we track seen volume IDs and only count each volume once for logical size/count.
// EC-encoded volumes are folded in via per-shard aggregation: every shard is
// node-local (not a replica), so shard sizes are summed across nodes; the
// per-volume file/delete counts carried on each shard message are deduped
// via max/sum so the aggregate doesn't double-count or drop after a volume
// is converted from regular to erasure coding.
func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) {
// Track which volumes we've already seen to deduplicate by volume ID
seenVolumes := make(map[volumeKey]bool)
ecVolumes := make(map[volumeKey]*ecVolumeAgg)
for _, dc := range t.DataCenterInfos {
for _, r := range dc.RackInfos {
for _, dn := range r.DataNodeInfos {
for _, diskInfo := range dn.DiskInfos {
for _, vi := range diskInfo.VolumeInfos {
c := vi.Collection
cif, found := collectionInfos[c]
if !found {
cif = &CollectionInfo{}
collectionInfos[c] = cif
}
// Always add to physical size (all replicas)
cif.PhysicalSize += float64(vi.Size)
// Check if we've already counted this volume for logical stats
key := volumeKey{collection: c, volumeId: vi.Id}
if seenVolumes[key] {
// Already counted this volume, skip logical stats
continue
}
seenVolumes[key] = true
// First time seeing this volume - add to logical stats
cif.Size += float64(vi.Size)
cif.FileCount += float64(vi.FileCount)
cif.DeleteCount += float64(vi.DeleteCount)
cif.DeletedByteCount += float64(vi.DeletedByteCount)
cif.VolumeCount++
}
for _, esi := range diskInfo.EcShardInfos {
c := esi.Collection
cif, found := collectionInfos[c]
if !found {
cif = &CollectionInfo{}
collectionInfos[c] = cif
}
// EC shards are node-local (no replication), so both
// physical and logical shard sizes sum across nodes
// without any dedupe. Logical size excludes parity
// shards; physical size includes them. Upstream OSS
// uses the fixed 10+4 ratio (dataShards=0 → default);
// forks with per-volume ratio metadata can pass the
// configured value here.
cif.PhysicalSize += float64(erasure_coding.EcShardsTotalSize(esi))
cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0))
key := volumeKey{collection: c, volumeId: esi.Id}
agg, ok := ecVolumes[key]
if !ok {
agg = &ecVolumeAgg{collection: c}
ecVolumes[key] = agg
cif.VolumeCount++
}
if esi.FileCount > agg.fileCount {
agg.fileCount = esi.FileCount
}
agg.deleteCount += esi.DeleteCount
}
}
}
}
}
// Fold deduped EC file/delete counts into each collection's totals.
for _, agg := range ecVolumes {
cif := collectionInfos[agg.collection]
if cif == nil {
continue
}
cif.FileCount += float64(agg.fileCount)
cif.DeleteCount += float64(agg.deleteCount)
}
}
weed/s3api/bucket_size_metrics_test.go
-211
View File
@@ -2,219 +2,8 @@ package s3api
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// TestCollectCollectionInfoFromTopologyEC verifies that EC-encoded volumes
// contribute to per-collection logical/physical size, file count, and volume
// count. Before this fix, encoding a volume to EC caused the bucket size
// metrics exported to Prometheus to drop to zero for that volume.
//
// Layout: one 10+4 EC volume in collection "crm-docs-storage", 14 shards of
// 1000 bytes each split across two nodes.
// - nodeA holds data shards 0..6 (7 * 1000 = 7000)
// - nodeB holds data shards 7..9 (3 * 1000 = 3000) and parity 10..13 (4 * 1000 = 4000)
//
// Expected:
// - PhysicalSize = 14 * 1000 = 14000
// - Size (logical, data shards) = 10 * 1000 = 10000
// - FileCount = 100 total - (2 + 3) local deletes = 95 is NOT what we check; the
// collector reports raw file_count and delete_count as separate gauges, so
// we assert FileCount = 100 (max across reporters) and DeleteCount = 5 (sum).
// - VolumeCount = 1 (one unique EC volume)
func TestCollectCollectionInfoFromTopologyEC(t *testing.T) {
nodeA := &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 42,
Collection: "crm-docs-storage",
EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
FileCount: 100,
DeleteCount: 2,
},
},
},
},
}
nodeB := &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 42,
Collection: "crm-docs-storage",
EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
FileCount: 100,
DeleteCount: 3,
},
},
},
},
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
},
},
},
},
}
got := make(map[string]*CollectionInfo)
collectCollectionInfoFromTopology(topo, got)
info, ok := got["crm-docs-storage"]
if !ok {
t.Fatalf("expected collection crm-docs-storage, got: %v", got)
}
if info.PhysicalSize != 14000 {
t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize)
}
if info.Size != 10000 {
t.Errorf("Size (logical): got %.0f, want 10000", info.Size)
}
if info.FileCount != 100 {
t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", info.FileCount)
}
if info.DeleteCount != 5 {
t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", info.DeleteCount)
}
if info.VolumeCount != 1 {
t.Errorf("VolumeCount: got %d, want 1", info.VolumeCount)
}
}
// TestCollectCollectionInfoFromTopologyMixed verifies that regular and EC
// volumes accumulate under the same collection without one clobbering the
// other, which is the state during an in-progress EC conversion.
func TestCollectCollectionInfoFromTopologyMixed(t *testing.T) {
node := &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
VolumeInfos: []*master_pb.VolumeInformationMessage{
{
Id: 1,
Collection: "bucket-mix",
Size: 5000,
FileCount: 50,
DeleteCount: 1,
DeletedByteCount: 100,
},
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 2,
Collection: "bucket-mix",
EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity
ShardSizes: []int64{3000, 3000, 3000},
FileCount: 80,
DeleteCount: 4,
},
},
},
},
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{node},
},
},
},
},
}
got := make(map[string]*CollectionInfo)
collectCollectionInfoFromTopology(topo, got)
info, ok := got["bucket-mix"]
if !ok {
t.Fatalf("expected collection bucket-mix, got: %v", got)
}
// Regular volume: 5000 physical + logical. EC shards: 9000 physical,
// 6000 logical (data shards 0 and 1).
if info.PhysicalSize != 5000+9000 {
t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize)
}
if info.Size != 5000+6000 {
t.Errorf("Size: got %.0f, want 11000", info.Size)
}
// LogicalSize drops the 100 bytes of un-vacuumed garbage on the regular
// volume; EC shards carry no DeletedByteCount here.
if info.LogicalSize() != 11000-100 {
t.Errorf("LogicalSize: got %.0f, want 10900", info.LogicalSize())
}
if info.FileCount != 50+80 {
t.Errorf("FileCount: got %.0f, want 130", info.FileCount)
}
if info.DeleteCount != 1+4 {
t.Errorf("DeleteCount: got %.0f, want 5", info.DeleteCount)
}
if info.VolumeCount != 2 {
t.Errorf("VolumeCount: got %d, want 2", info.VolumeCount)
}
}
// TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe verifies that a
// slow shard holder reporting file_count=0 (because it has not yet finished
// loading .ecx) does not pin the per-volume FileCount at 0.
func TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe(t *testing.T) {
makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 11,
Collection: "bucket-b",
EcIndexBits: bits,
ShardSizes: sizes,
FileCount: fileCount,
},
},
},
},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0),
makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6),
},
},
},
},
},
}
got := make(map[string]*CollectionInfo)
collectCollectionInfoFromTopology(topo, got)
info, ok := got["bucket-b"]
if !ok {
t.Fatalf("expected collection bucket-b, got: %v", got)
}
if info.FileCount != 6 {
t.Errorf("FileCount: got %.0f, want 6 (max across reporters)", info.FileCount)
}
}
// TestCollectionInfoLogicalSize verifies logical size excludes un-vacuumed
// garbage and never goes negative. Quota enforcement runs on this value so a
// bucket full of tombstones is not flipped read-only while its live data is