From 5f4d05b1035248c832341707b451ad70a7c3000b Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:29:21 +0530 Subject: [PATCH] admin: measure volume server usage like its capacity (#12306) The volume server views summed the logical size of the volumes into DiskUsage while DiskCapacity came from the filesystem statfs size. On a compressing filesystem the logical sum exceeds the physical capacity, so the server details popup derived a negative available space and rendered it as "NaN undefined". Pair each disk's capacity with the usage measured the same way, using the statfs numbers the volume server already heartbeats, as CollectTierStats does. Volume servers without statfs numbers keep the previous slot-based capacity and logical bytes, and EC shard bytes are only counted there instead of on top of the filesystem usage. --- weed/admin/dash/cluster_topology.go | 42 ++++++-- weed/admin/dash/types.go | 4 +- weed/admin/dash/volume_management.go | 25 ++--- weed/admin/dash/volume_management_test.go | 125 ++++++++++++++++++++++ 4 files changed, 169 insertions(+), 27 deletions(-) diff --git a/weed/admin/dash/cluster_topology.go b/weed/admin/dash/cluster_topology.go index 21759dd34a..dd39d0fa16 100644 --- a/weed/admin/dash/cluster_topology.go +++ b/weed/admin/dash/cluster_topology.go @@ -109,6 +109,27 @@ func (s *AdminServer) fetchPublicUrlMap() map[string]string { return publicUrls } +// diskCapacityAndUsage returns the physical capacity of one disk together with +// the bytes in use on it, measured the same way. The bytes in use are the +// statfs numbers the volume server heartbeats, DiskTotalBytes-DiskFreeBytes, +// which is the same measurement CollectTierStats uses. The logical size of the +// volumes is not comparable with the physical capacity: a compressing +// filesystem (ZFS, Btrfs) stores fewer bytes than the volumes report, so +// counting logical bytes as usage made the admin UI show usage above capacity +// and derive a negative available space. Volume servers that do not report +// statfs numbers keep the previous behaviour: slot-based capacity and the +// logical bytes the disk holds. +func diskCapacityAndUsage(diskInfo *master_pb.DiskInfo, volumeSizeLimitMb uint64, logicalBytes int64) (capacity int64, usage int64) { + if diskInfo.DiskTotalBytes > 0 { + capacity = int64(diskInfo.DiskTotalBytes) + if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes { + usage = int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes) + } + return capacity, usage + } + return diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024, logicalBytes +} + // getTopologyViaGRPC gets topology using gRPC (original method) func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { // Fetch public URL mapping from master HTTP API @@ -150,28 +171,22 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { var totalMaxVolumes int64 var totalSize int64 var remoteSize int64 - // Prefer the real physical disk capacity the volume server - // reports per disk; the slot-based estimate overstates capacity - // when maxVolumeCount is configured higher than the disk holds. var diskCapacity int64 + var diskUsage int64 for _, diskInfo := range node.DiskInfos { totalVolumes += diskInfo.VolumeCount totalMaxVolumes += diskInfo.MaxVolumeCount - if diskInfo.DiskTotalBytes > 0 { - diskCapacity += int64(diskInfo.DiskTotalBytes) - } else { - diskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024 - } // A remote-tiered volume reports its cloud object's // size; keep those bytes out of the local disk usage // that is compared against diskCapacity. + var diskLocalSize int64 for _, volInfo := range diskInfo.VolumeInfos { if volInfo.RemoteStorageName != "" { remoteSize += int64(volInfo.Size) } else { - totalSize += int64(volInfo.Size) + diskLocalSize += int64(volInfo.Size) } } @@ -179,9 +194,14 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { // across nodes gives the physical footprint. for _, ecShardInfo := range diskInfo.EcShardInfos { for _, shardSize := range ecShardInfo.ShardSizes { - totalSize += shardSize + diskLocalSize += shardSize } } + totalSize += diskLocalSize + + capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMb, diskLocalSize) + diskCapacity += capacity + diskUsage += usage } // Look up PublicUrl from master HTTP API @@ -203,7 +223,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { PublicURL: publicUrl, Volumes: int(totalVolumes), MaxVolumes: int(totalMaxVolumes), - DiskUsage: totalSize, + DiskUsage: diskUsage, DiskCapacity: diskCapacity, LastHeartbeat: time.Now(), RemoteSize: remoteSize, diff --git a/weed/admin/dash/types.go b/weed/admin/dash/types.go index f884383255..945874fc84 100644 --- a/weed/admin/dash/types.go +++ b/weed/admin/dash/types.go @@ -50,8 +50,8 @@ type VolumeServer struct { Version string `json:"version"` Volumes int `json:"volumes"` MaxVolumes int `json:"max_volumes"` - DiskUsage int64 `json:"disk_usage"` - DiskCapacity int64 `json:"disk_capacity"` + DiskUsage int64 `json:"disk_usage"` // statfs bytes in use, comparable with DiskCapacity + DiskCapacity int64 `json:"disk_capacity"` // statfs capacity, or the slot-based estimate without statfs LastHeartbeat time.Time `json:"last_heartbeat"` // EC shard information diff --git a/weed/admin/dash/volume_management.go b/weed/admin/dash/volume_management.go index f4905b1457..2d787871bd 100644 --- a/weed/admin/dash/volume_management.go +++ b/weed/admin/dash/volume_management.go @@ -533,33 +533,31 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro // Process disk information for _, diskInfo := range node.DiskInfos { vs.MaxVolumes += int(diskInfo.MaxVolumeCount) - // Prefer the real physical disk capacity the volume server - // reports; the slot-based estimate overstates capacity when - // maxVolumeCount is configured higher than the disk holds. - if diskInfo.DiskTotalBytes > 0 { - vs.DiskCapacity += int64(diskInfo.DiskTotalBytes) - } else { - vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024 - } - // Count regular volumes and calculate disk usage. - // A remote-tiered volume reports its cloud object's - // size; keep those bytes out of the local disk usage - // that is compared against DiskCapacity. + // Count regular volumes. A remote-tiered volume + // reports its cloud object's size; keep those bytes + // out of the local disk usage that is compared + // against DiskCapacity. + var diskLocalSize int64 for _, volInfo := range diskInfo.VolumeInfos { vs.Volumes++ if volInfo.RemoteStorageName != "" { vs.RemoteSize += int64(volInfo.Size) } else { - vs.DiskUsage += int64(volInfo.Size) + diskLocalSize += int64(volInfo.Size) } } // Accumulate EC shard information across all disks for this volume server for _, ecShardInfo := range diskInfo.EcShardInfos { + diskLocalSize += erasure_coding.EcShardsTotalSize(ecShardInfo) volumeId := ecShardInfo.Id ecShardAccumulator[volumeId] = append(ecShardAccumulator[volumeId], ecShardInfo) } + + capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMB, diskLocalSize) + vs.DiskCapacity += capacity + vs.DiskUsage += usage } // Process accumulated EC shard information per volume @@ -601,7 +599,6 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro if shardSize, exists := allShardSizes[shardId]; exists { ecInfo.ShardSizes[int(shardId)] = shardSize ecInfo.TotalSize += shardSize - vs.DiskUsage += shardSize // Add EC shard size to total disk usage } } diff --git a/weed/admin/dash/volume_management_test.go b/weed/admin/dash/volume_management_test.go index 7381dec71f..0f7fd04f40 100644 --- a/weed/admin/dash/volume_management_test.go +++ b/weed/admin/dash/volume_management_test.go @@ -3,6 +3,8 @@ package dash import ( "context" "net" + "net/http" + "net/http/httptest" "testing" "time" @@ -157,3 +159,126 @@ func TestVolumeReplicaAddress(t *testing.T) { _, err = volumeReplicaAddress(nil, 7, "node-b") require.Error(t, err) } + +// newVolumeServersTestAdmin wires an AdminServer to a fake master that returns +// topology, so the cluster volume server views can be built without a real +// cluster. +func newVolumeServersTestAdmin(t *testing.T, topology *master_pb.TopologyInfo) *AdminServer { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, &volumeAccessTestMaster{topology: topology}) + go grpcServer.Serve(listener) + t.Cleanup(grpcServer.Stop) + + // The master HTTP API only supplies PublicUrl, which disk accounting does + // not use; answer it locally so the test never leaves the machine. + httpServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"Topology":{"DataCenters":[]}}`)) + })) + httpServer.Start() + t.Cleanup(httpServer.Close) + + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + address := pb.NewServerAddress( + "127.0.0.1", + httpServer.Listener.Addr().(*net.TCPAddr).Port, + listener.Addr().(*net.TCPAddr).Port, + ) + discovery := pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"master": address}) + masterClient := wdclient.NewMasterClient(dialOption, "", "admin", "", "", "", *discovery) + go masterClient.KeepConnectedToMaster(t.Context()) + + admin := &AdminServer{masterClient: masterClient, grpcDialOption: dialOption} + // Versions come from the VolumeServerStatus fan-out, which is unrelated to + // disk accounting. A fresh cache keeps the test from dialing volume servers. + admin.volumeVersionCache = map[string]string{"node-a": "4.51"} + admin.lastVersionFetch = time.Now() + return admin +} + +// TestGetClusterVolumeServersDiskAccounting pins the volume server usage to the +// same measurement as the capacity it is shown against. Usage used to sum the +// logical volume sizes, which on a compressing filesystem exceeds the physical +// capacity; the admin UI then derived a negative available space and rendered +// it as "NaN undefined". +func TestGetClusterVolumeServersDiskAccounting(t *testing.T) { + for _, tc := range []struct { + name string + disk *master_pb.DiskInfo + wantCapacity int64 + wantUsage int64 + }{ + { + name: "compressed filesystem reports statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 5_700_000_000_000, + DiskFreeBytes: 1_500_000_000_000, + VolumeInfos: []*master_pb.VolumeInformationMessage{ + {Id: 1, Size: 3_000_000_000_000}, + {Id: 2, Size: 3_020_000_000_000}, + }, + }, + wantCapacity: 5_700_000_000_000, + wantUsage: 4_200_000_000_000, + }, + { + name: "plain filesystem reports statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 1_000_000_000_000, + DiskFreeBytes: 400_000_000_000, + VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 1, Size: 590_000_000_000}}, + }, + wantCapacity: 1_000_000_000_000, + wantUsage: 600_000_000_000, + }, + { + name: "ec shards are not added on top of statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 1_000_000_000_000, + DiskFreeBytes: 300_000_000_000, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 3, EcIndexBits: 0b11, ShardSizes: []int64{100, 200}}, + }, + }, + wantCapacity: 1_000_000_000_000, + wantUsage: 700_000_000_000, + }, + { + name: "without statfs numbers falls back to logical size", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 10, + VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 4, Size: 30_000_000_000}}, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{300}}, + }, + }, + // 10 slots at the 30000 MiB default volume size limit. + wantCapacity: 10 * 30000 * 1024 * 1024, + wantUsage: 30_000_000_000 + 300, + }, + } { + t.Run(tc.name, func(t *testing.T) { + admin := newVolumeServersTestAdmin(t, volumeAccessTestTopology(&master_pb.DataNodeInfo{ + Id: "node-a", + Address: "127.0.0.1:8080", + GrpcPort: 18080, + DiskInfos: map[string]*master_pb.DiskInfo{"hdd": tc.disk}, + })) + + data, err := admin.GetClusterVolumeServers() + require.NoError(t, err) + require.Len(t, data.VolumeServers, 1) + server := data.VolumeServers[0] + require.Equal(t, tc.wantCapacity, server.DiskCapacity) + require.Equal(t, tc.wantUsage, server.DiskUsage) + require.LessOrEqual(t, server.DiskUsage, server.DiskCapacity, + "usage above capacity makes the advertised available space negative") + }) + } +}