admin: show disk usage on the dashboard instead of the data size (#12466)

* admin: show disk usage on the dashboard instead of the data size

The dashboard's Total Size card summed the volume and EC shard sizes the
volume servers report. That leaves out deleted needles waiting for vacuum,
EC generations kept after a vacuum, and anything else on the disks, so on a
cluster with a lot of EC vacuum activity it showed far less than the disks
held.

The card now shows the bytes in use on the volume servers' disks, the same
statfs numbers the volume server list uses, with the disks' capacity, and
its trend line follows that number. Servers that report no statfs numbers
count their data size, as on the volume server list.

Claude-Session: https://claude.ai/code/session_011UpvafcfUuC2mLxNAXLQJi

* admin: compute the disk usage percentage in float64

current*100 overflows int64 past about 92 PB of used space, so the
dashboard card showed a negative percentage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: size the volume server disk bar with calculatePercent64

int(host.DiskUsage) truncates byte counts on 32-bit builds, so the bar
width was wrong for any disk over 2 GB there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: test the cluster disk totals across statfs and fallback servers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-06 19:25:10 -08:00
1 parent fd81a50c57
commit 7beac0e8ea
11 files changed
+559 -425

No files matched your search

+4
View File
@@ -25,6 +25,8 @@ type AdminData struct {
TotalVolumes int `json:"total_volumes"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
DiskUsage int64 `json:"disk_usage"`
DiskCapacity int64 `json:"disk_capacity"`
VolumeSizeLimitMB uint64 `json:"volume_size_limit_mb"`
MasterNodes []MasterNode `json:"master_nodes"`
VolumeServers []VolumeServer `json:"volume_servers"`
@@ -240,6 +242,8 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
TotalVolumes: topology.TotalVolumes,
TotalChunks: topology.TotalChunks,
TotalSize: topology.TotalSize,
DiskUsage: topology.DiskUsage,
DiskCapacity: topology.DiskCapacity,
VolumeSizeLimitMB: volumeSizeLimitMB,
MasterNodes: masterNodes,
VolumeServers: volumeServersData.VolumeServers,
+2
View File
@@ -235,6 +235,8 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
// TotalSize is the logical data size, wherever the
// bytes live, so remote-tiered volumes still count.
topology.TotalSize += totalSize + remoteSize
topology.DiskUsage += diskUsage
topology.DiskCapacity += diskCapacity
}
dataCenter.Racks = append(dataCenter.Racks, rackObj)
+1 -1
View File
@@ -60,7 +60,7 @@ func (s *AdminServer) recordDashboardSample() {
t: time.Now(),
volumes: float64(topology.TotalVolumes),
ecShards: float64(ecShards),
diskUsed: float64(topology.TotalSize),
diskUsed: float64(topology.DiskUsage),
chunks: float64(topology.TotalChunks),
}
if s.maintenanceManager != nil {
+8 -3
View File
@@ -18,9 +18,14 @@ type ClusterTopology struct {
TotalVolumes int `json:"total_volumes"`
// TotalChunks counts chunks stored in volumes, not filer entries: a file
// is split into one or more chunks.
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
UpdatedAt time.Time `json:"updated_at"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
// DiskUsage and DiskCapacity sum the volume servers' DiskUsage and
// DiskCapacity: the bytes in use on their disks, parity, deleted needles
// and retained EC generations included, and the disks' size.
DiskUsage int64 `json:"disk_usage"`
DiskCapacity int64 `json:"disk_capacity"`
UpdatedAt time.Time `json:"updated_at"`
// TierStats breaks volumes and EC shards down by storage tier: local
// disk types plus one entry per remote storage holding tiered volumes.
TierStats []TierStats `json:"tier_stats"`
+32
View File
@@ -282,3 +282,35 @@ func TestGetClusterVolumeServersDiskAccounting(t *testing.T) {
})
}
}
// The dashboard totals sum each server's disk usage and capacity, statfs or
// fallback, while TotalSize stays the logical data size.
func TestClusterTopologyDiskTotals(t *testing.T) {
admin := newVolumeServersTestAdmin(t, volumeAccessTestTopology(
&master_pb.DataNodeInfo{
Id: "node-a",
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {
MaxVolumeCount: 100,
DiskTotalBytes: 1_000_000_000_000,
DiskFreeBytes: 400_000_000_000,
VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 1, Size: 590_000_000_000}},
}},
},
&master_pb.DataNodeInfo{
Id: "node-b",
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {
MaxVolumeCount: 10,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 2, Size: 30_000_000_000},
{Id: 3, Size: 5_000_000_000, RemoteStorageName: "s3"},
},
}},
},
))
topology, err := admin.GetClusterTopology()
require.NoError(t, err)
require.Equal(t, int64(600_000_000_000+30_000_000_000), topology.DiskUsage)
require.Equal(t, int64(1_000_000_000_000+10*30000*1024*1024), topology.DiskCapacity)
require.Equal(t, int64(590_000_000_000+30_000_000_000+5_000_000_000), topology.TotalSize)
}