admin: measure volume server usage like its capacity (#12306)

The volume server views summed the logical size of the volumes into
DiskUsage while DiskCapacity came from the filesystem statfs size. On a
compressing filesystem the logical sum exceeds the physical capacity, so
the server details popup derived a negative available space and rendered
it as "NaN undefined".

Pair each disk's capacity with the usage measured the same way, using the
statfs numbers the volume server already heartbeats, as CollectTierStats
does. Volume servers without statfs numbers keep the previous slot-based
capacity and logical bytes, and EC shard bytes are only counted there
instead of on top of the filesystem usage.

weed/admin/dash/cluster_topology.go

@@ -109,6 +109,27 @@ func (s *AdminServer) fetchPublicUrlMap() map[string]string {
 	return publicUrls
 }
 
+// diskCapacityAndUsage returns the physical capacity of one disk together with
+// the bytes in use on it, measured the same way. The bytes in use are the
+// statfs numbers the volume server heartbeats, DiskTotalBytes-DiskFreeBytes,
+// which is the same measurement CollectTierStats uses. The logical size of the
+// volumes is not comparable with the physical capacity: a compressing
+// filesystem (ZFS, Btrfs) stores fewer bytes than the volumes report, so
+// counting logical bytes as usage made the admin UI show usage above capacity
+// and derive a negative available space. Volume servers that do not report
+// statfs numbers keep the previous behaviour: slot-based capacity and the
+// logical bytes the disk holds.
+func diskCapacityAndUsage(diskInfo *master_pb.DiskInfo, volumeSizeLimitMb uint64, logicalBytes int64) (capacity int64, usage int64) {
+	if diskInfo.DiskTotalBytes > 0 {
+		capacity = int64(diskInfo.DiskTotalBytes)
+		if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes {
+			usage = int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes)
+		}
+		return capacity, usage
+	}
+	return diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024, logicalBytes
+}
+
 // getTopologyViaGRPC gets topology using gRPC (original method)
 func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
 	// Fetch public URL mapping from master HTTP API
@@ -150,28 +171,22 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
 						var totalMaxVolumes int64
 						var totalSize int64
 						var remoteSize int64
-						// Prefer the real physical disk capacity the volume server
-						// reports per disk; the slot-based estimate overstates capacity
-						// when maxVolumeCount is configured higher than the disk holds.
 						var diskCapacity int64
+						var diskUsage int64
 
 						for _, diskInfo := range node.DiskInfos {
 							totalVolumes += diskInfo.VolumeCount
 							totalMaxVolumes += diskInfo.MaxVolumeCount
-							if diskInfo.DiskTotalBytes > 0 {
-								diskCapacity += int64(diskInfo.DiskTotalBytes)
-							} else {
-								diskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024
-							}
 
 							// A remote-tiered volume reports its cloud object's
 							// size; keep those bytes out of the local disk usage
 							// that is compared against diskCapacity.
+							var diskLocalSize int64
 							for _, volInfo := range diskInfo.VolumeInfos {
 								if volInfo.RemoteStorageName != "" {
 									remoteSize += int64(volInfo.Size)
 								} else {
-									totalSize += int64(volInfo.Size)
+									diskLocalSize += int64(volInfo.Size)
 								}
 							}
 
@@ -179,9 +194,14 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
 							// across nodes gives the physical footprint.
 							for _, ecShardInfo := range diskInfo.EcShardInfos {
 								for _, shardSize := range ecShardInfo.ShardSizes {
-									totalSize += shardSize
+									diskLocalSize += shardSize
 								}
 							}
+							totalSize += diskLocalSize
+
+							capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMb, diskLocalSize)
+							diskCapacity += capacity
+							diskUsage += usage
 						}
 
 						// Look up PublicUrl from master HTTP API
@@ -203,7 +223,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
 							PublicURL:     publicUrl,
 							Volumes:       int(totalVolumes),
 							MaxVolumes:    int(totalMaxVolumes),
-							DiskUsage:     totalSize,
+							DiskUsage:     diskUsage,
 							DiskCapacity:  diskCapacity,
 							LastHeartbeat: time.Now(),
 							RemoteSize:    remoteSize,

weed/admin/dash/types.go

@@ -50,8 +50,8 @@ type VolumeServer struct {
 	Version       string    `json:"version"`
 	Volumes       int       `json:"volumes"`
 	MaxVolumes    int       `json:"max_volumes"`
-	DiskUsage     int64     `json:"disk_usage"`
-	DiskCapacity  int64     `json:"disk_capacity"`
+	DiskUsage     int64     `json:"disk_usage"`    // statfs bytes in use, comparable with DiskCapacity
+	DiskCapacity  int64     `json:"disk_capacity"` // statfs capacity, or the slot-based estimate without statfs
 	LastHeartbeat time.Time `json:"last_heartbeat"`
 
 	// EC shard information

weed/admin/dash/volume_management.go

@@ -533,33 +533,31 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
 						// Process disk information
 						for _, diskInfo := range node.DiskInfos {
 							vs.MaxVolumes += int(diskInfo.MaxVolumeCount)
-							// Prefer the real physical disk capacity the volume server
-							// reports; the slot-based estimate overstates capacity when
-							// maxVolumeCount is configured higher than the disk holds.
-							if diskInfo.DiskTotalBytes > 0 {
-								vs.DiskCapacity += int64(diskInfo.DiskTotalBytes)
-							} else {
-								vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024
-							}
 
-							// Count regular volumes and calculate disk usage.
-							// A remote-tiered volume reports its cloud object's
-							// size; keep those bytes out of the local disk usage
-							// that is compared against DiskCapacity.
+							// Count regular volumes. A remote-tiered volume
+							// reports its cloud object's size; keep those bytes
+							// out of the local disk usage that is compared
+							// against DiskCapacity.
+							var diskLocalSize int64
 							for _, volInfo := range diskInfo.VolumeInfos {
 								vs.Volumes++
 								if volInfo.RemoteStorageName != "" {
 									vs.RemoteSize += int64(volInfo.Size)
 								} else {
-									vs.DiskUsage += int64(volInfo.Size)
+									diskLocalSize += int64(volInfo.Size)
 								}
 							}
 
 							// Accumulate EC shard information across all disks for this volume server
 							for _, ecShardInfo := range diskInfo.EcShardInfos {
+								diskLocalSize += erasure_coding.EcShardsTotalSize(ecShardInfo)
 								volumeId := ecShardInfo.Id
 								ecShardAccumulator[volumeId] = append(ecShardAccumulator[volumeId], ecShardInfo)
 							}
+
+							capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMB, diskLocalSize)
+							vs.DiskCapacity += capacity
+							vs.DiskUsage += usage
 						}
 
 						// Process accumulated EC shard information per volume
@@ -601,7 +599,6 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
 								if shardSize, exists := allShardSizes[shardId]; exists {
 									ecInfo.ShardSizes[int(shardId)] = shardSize
 									ecInfo.TotalSize += shardSize
-									vs.DiskUsage += shardSize // Add EC shard size to total disk usage
 								}
 							}
 

weed/admin/dash/volume_management_test.go

@@ -3,6 +3,8 @@ package dash
 import (
 	"context"
 	"net"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -157,3 +159,126 @@ func TestVolumeReplicaAddress(t *testing.T) {
 	_, err = volumeReplicaAddress(nil, 7, "node-b")
 	require.Error(t, err)
 }
+
+// newVolumeServersTestAdmin wires an AdminServer to a fake master that returns
+// topology, so the cluster volume server views can be built without a real
+// cluster.
+func newVolumeServersTestAdmin(t *testing.T, topology *master_pb.TopologyInfo) *AdminServer {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, &volumeAccessTestMaster{topology: topology})
+	go grpcServer.Serve(listener)
+	t.Cleanup(grpcServer.Stop)
+
+	// The master HTTP API only supplies PublicUrl, which disk accounting does
+	// not use; answer it locally so the test never leaves the machine.
+	httpServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"Topology":{"DataCenters":[]}}`))
+	}))
+	httpServer.Start()
+	t.Cleanup(httpServer.Close)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	address := pb.NewServerAddress(
+		"127.0.0.1",
+		httpServer.Listener.Addr().(*net.TCPAddr).Port,
+		listener.Addr().(*net.TCPAddr).Port,
+	)
+	discovery := pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"master": address})
+	masterClient := wdclient.NewMasterClient(dialOption, "", "admin", "", "", "", *discovery)
+	go masterClient.KeepConnectedToMaster(t.Context())
+
+	admin := &AdminServer{masterClient: masterClient, grpcDialOption: dialOption}
+	// Versions come from the VolumeServerStatus fan-out, which is unrelated to
+	// disk accounting. A fresh cache keeps the test from dialing volume servers.
+	admin.volumeVersionCache = map[string]string{"node-a": "4.51"}
+	admin.lastVersionFetch = time.Now()
+	return admin
+}
+
+// TestGetClusterVolumeServersDiskAccounting pins the volume server usage to the
+// same measurement as the capacity it is shown against. Usage used to sum the
+// logical volume sizes, which on a compressing filesystem exceeds the physical
+// capacity; the admin UI then derived a negative available space and rendered
+// it as "NaN undefined".
+func TestGetClusterVolumeServersDiskAccounting(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		disk         *master_pb.DiskInfo
+		wantCapacity int64
+		wantUsage    int64
+	}{
+		{
+			name: "compressed filesystem reports statfs usage",
+			disk: &master_pb.DiskInfo{
+				MaxVolumeCount: 100,
+				DiskTotalBytes: 5_700_000_000_000,
+				DiskFreeBytes:  1_500_000_000_000,
+				VolumeInfos: []*master_pb.VolumeInformationMessage{
+					{Id: 1, Size: 3_000_000_000_000},
+					{Id: 2, Size: 3_020_000_000_000},
+				},
+			},
+			wantCapacity: 5_700_000_000_000,
+			wantUsage:    4_200_000_000_000,
+		},
+		{
+			name: "plain filesystem reports statfs usage",
+			disk: &master_pb.DiskInfo{
+				MaxVolumeCount: 100,
+				DiskTotalBytes: 1_000_000_000_000,
+				DiskFreeBytes:  400_000_000_000,
+				VolumeInfos:    []*master_pb.VolumeInformationMessage{{Id: 1, Size: 590_000_000_000}},
+			},
+			wantCapacity: 1_000_000_000_000,
+			wantUsage:    600_000_000_000,
+		},
+		{
+			name: "ec shards are not added on top of statfs usage",
+			disk: &master_pb.DiskInfo{
+				MaxVolumeCount: 100,
+				DiskTotalBytes: 1_000_000_000_000,
+				DiskFreeBytes:  300_000_000_000,
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{Id: 3, EcIndexBits: 0b11, ShardSizes: []int64{100, 200}},
+				},
+			},
+			wantCapacity: 1_000_000_000_000,
+			wantUsage:    700_000_000_000,
+		},
+		{
+			name: "without statfs numbers falls back to logical size",
+			disk: &master_pb.DiskInfo{
+				MaxVolumeCount: 10,
+				VolumeInfos:    []*master_pb.VolumeInformationMessage{{Id: 4, Size: 30_000_000_000}},
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{300}},
+				},
+			},
+			// 10 slots at the 30000 MiB default volume size limit.
+			wantCapacity: 10 * 30000 * 1024 * 1024,
+			wantUsage:    30_000_000_000 + 300,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			admin := newVolumeServersTestAdmin(t, volumeAccessTestTopology(&master_pb.DataNodeInfo{
+				Id:        "node-a",
+				Address:   "127.0.0.1:8080",
+				GrpcPort:  18080,
+				DiskInfos: map[string]*master_pb.DiskInfo{"hdd": tc.disk},
+			}))
+
+			data, err := admin.GetClusterVolumeServers()
+			require.NoError(t, err)
+			require.Len(t, data.VolumeServers, 1)
+			server := data.VolumeServers[0]
+			require.Equal(t, tc.wantCapacity, server.DiskCapacity)
+			require.Equal(t, tc.wantUsage, server.DiskUsage)
+			require.LessOrEqual(t, server.DiskUsage, server.DiskCapacity,
+				"usage above capacity makes the advertised available space negative")
+		})
+	}
+}