diff --git a/weed/admin/dash/cluster_topology.go b/weed/admin/dash/cluster_topology.go index 21759dd34a..dd39d0fa16 100644 --- a/weed/admin/dash/cluster_topology.go +++ b/weed/admin/dash/cluster_topology.go @@ -109,6 +109,27 @@ func (s *AdminServer) fetchPublicUrlMap() map[string]string { return publicUrls } +// diskCapacityAndUsage returns the physical capacity of one disk together with +// the bytes in use on it, measured the same way. The bytes in use are the +// statfs numbers the volume server heartbeats, DiskTotalBytes-DiskFreeBytes, +// which is the same measurement CollectTierStats uses. The logical size of the +// volumes is not comparable with the physical capacity: a compressing +// filesystem (ZFS, Btrfs) stores fewer bytes than the volumes report, so +// counting logical bytes as usage made the admin UI show usage above capacity +// and derive a negative available space. Volume servers that do not report +// statfs numbers keep the previous behaviour: slot-based capacity and the +// logical bytes the disk holds. +func diskCapacityAndUsage(diskInfo *master_pb.DiskInfo, volumeSizeLimitMb uint64, logicalBytes int64) (capacity int64, usage int64) { + if diskInfo.DiskTotalBytes > 0 { + capacity = int64(diskInfo.DiskTotalBytes) + if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes { + usage = int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes) + } + return capacity, usage + } + return diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024, logicalBytes +} + // getTopologyViaGRPC gets topology using gRPC (original method) func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { // Fetch public URL mapping from master HTTP API @@ -150,28 +171,22 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { var totalMaxVolumes int64 var totalSize int64 var remoteSize int64 - // Prefer the real physical disk capacity the volume server - // reports per disk; the slot-based estimate overstates capacity - // when maxVolumeCount is configured higher than the disk holds. var diskCapacity int64 + var diskUsage int64 for _, diskInfo := range node.DiskInfos { totalVolumes += diskInfo.VolumeCount totalMaxVolumes += diskInfo.MaxVolumeCount - if diskInfo.DiskTotalBytes > 0 { - diskCapacity += int64(diskInfo.DiskTotalBytes) - } else { - diskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024 - } // A remote-tiered volume reports its cloud object's // size; keep those bytes out of the local disk usage // that is compared against diskCapacity. + var diskLocalSize int64 for _, volInfo := range diskInfo.VolumeInfos { if volInfo.RemoteStorageName != "" { remoteSize += int64(volInfo.Size) } else { - totalSize += int64(volInfo.Size) + diskLocalSize += int64(volInfo.Size) } } @@ -179,9 +194,14 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { // across nodes gives the physical footprint. for _, ecShardInfo := range diskInfo.EcShardInfos { for _, shardSize := range ecShardInfo.ShardSizes { - totalSize += shardSize + diskLocalSize += shardSize } } + totalSize += diskLocalSize + + capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMb, diskLocalSize) + diskCapacity += capacity + diskUsage += usage } // Look up PublicUrl from master HTTP API @@ -203,7 +223,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error { PublicURL: publicUrl, Volumes: int(totalVolumes), MaxVolumes: int(totalMaxVolumes), - DiskUsage: totalSize, + DiskUsage: diskUsage, DiskCapacity: diskCapacity, LastHeartbeat: time.Now(), RemoteSize: remoteSize, diff --git a/weed/admin/dash/types.go b/weed/admin/dash/types.go index f884383255..945874fc84 100644 --- a/weed/admin/dash/types.go +++ b/weed/admin/dash/types.go @@ -50,8 +50,8 @@ type VolumeServer struct { Version string `json:"version"` Volumes int `json:"volumes"` MaxVolumes int `json:"max_volumes"` - DiskUsage int64 `json:"disk_usage"` - DiskCapacity int64 `json:"disk_capacity"` + DiskUsage int64 `json:"disk_usage"` // statfs bytes in use, comparable with DiskCapacity + DiskCapacity int64 `json:"disk_capacity"` // statfs capacity, or the slot-based estimate without statfs LastHeartbeat time.Time `json:"last_heartbeat"` // EC shard information diff --git a/weed/admin/dash/volume_management.go b/weed/admin/dash/volume_management.go index f4905b1457..2d787871bd 100644 --- a/weed/admin/dash/volume_management.go +++ b/weed/admin/dash/volume_management.go @@ -533,33 +533,31 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro // Process disk information for _, diskInfo := range node.DiskInfos { vs.MaxVolumes += int(diskInfo.MaxVolumeCount) - // Prefer the real physical disk capacity the volume server - // reports; the slot-based estimate overstates capacity when - // maxVolumeCount is configured higher than the disk holds. - if diskInfo.DiskTotalBytes > 0 { - vs.DiskCapacity += int64(diskInfo.DiskTotalBytes) - } else { - vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024 - } - // Count regular volumes and calculate disk usage. - // A remote-tiered volume reports its cloud object's - // size; keep those bytes out of the local disk usage - // that is compared against DiskCapacity. + // Count regular volumes. A remote-tiered volume + // reports its cloud object's size; keep those bytes + // out of the local disk usage that is compared + // against DiskCapacity. + var diskLocalSize int64 for _, volInfo := range diskInfo.VolumeInfos { vs.Volumes++ if volInfo.RemoteStorageName != "" { vs.RemoteSize += int64(volInfo.Size) } else { - vs.DiskUsage += int64(volInfo.Size) + diskLocalSize += int64(volInfo.Size) } } // Accumulate EC shard information across all disks for this volume server for _, ecShardInfo := range diskInfo.EcShardInfos { + diskLocalSize += erasure_coding.EcShardsTotalSize(ecShardInfo) volumeId := ecShardInfo.Id ecShardAccumulator[volumeId] = append(ecShardAccumulator[volumeId], ecShardInfo) } + + capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMB, diskLocalSize) + vs.DiskCapacity += capacity + vs.DiskUsage += usage } // Process accumulated EC shard information per volume @@ -601,7 +599,6 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro if shardSize, exists := allShardSizes[shardId]; exists { ecInfo.ShardSizes[int(shardId)] = shardSize ecInfo.TotalSize += shardSize - vs.DiskUsage += shardSize // Add EC shard size to total disk usage } } diff --git a/weed/admin/dash/volume_management_test.go b/weed/admin/dash/volume_management_test.go index 7381dec71f..0f7fd04f40 100644 --- a/weed/admin/dash/volume_management_test.go +++ b/weed/admin/dash/volume_management_test.go @@ -3,6 +3,8 @@ package dash import ( "context" "net" + "net/http" + "net/http/httptest" "testing" "time" @@ -157,3 +159,126 @@ func TestVolumeReplicaAddress(t *testing.T) { _, err = volumeReplicaAddress(nil, 7, "node-b") require.Error(t, err) } + +// newVolumeServersTestAdmin wires an AdminServer to a fake master that returns +// topology, so the cluster volume server views can be built without a real +// cluster. +func newVolumeServersTestAdmin(t *testing.T, topology *master_pb.TopologyInfo) *AdminServer { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, &volumeAccessTestMaster{topology: topology}) + go grpcServer.Serve(listener) + t.Cleanup(grpcServer.Stop) + + // The master HTTP API only supplies PublicUrl, which disk accounting does + // not use; answer it locally so the test never leaves the machine. + httpServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"Topology":{"DataCenters":[]}}`)) + })) + httpServer.Start() + t.Cleanup(httpServer.Close) + + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + address := pb.NewServerAddress( + "127.0.0.1", + httpServer.Listener.Addr().(*net.TCPAddr).Port, + listener.Addr().(*net.TCPAddr).Port, + ) + discovery := pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"master": address}) + masterClient := wdclient.NewMasterClient(dialOption, "", "admin", "", "", "", *discovery) + go masterClient.KeepConnectedToMaster(t.Context()) + + admin := &AdminServer{masterClient: masterClient, grpcDialOption: dialOption} + // Versions come from the VolumeServerStatus fan-out, which is unrelated to + // disk accounting. A fresh cache keeps the test from dialing volume servers. + admin.volumeVersionCache = map[string]string{"node-a": "4.51"} + admin.lastVersionFetch = time.Now() + return admin +} + +// TestGetClusterVolumeServersDiskAccounting pins the volume server usage to the +// same measurement as the capacity it is shown against. Usage used to sum the +// logical volume sizes, which on a compressing filesystem exceeds the physical +// capacity; the admin UI then derived a negative available space and rendered +// it as "NaN undefined". +func TestGetClusterVolumeServersDiskAccounting(t *testing.T) { + for _, tc := range []struct { + name string + disk *master_pb.DiskInfo + wantCapacity int64 + wantUsage int64 + }{ + { + name: "compressed filesystem reports statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 5_700_000_000_000, + DiskFreeBytes: 1_500_000_000_000, + VolumeInfos: []*master_pb.VolumeInformationMessage{ + {Id: 1, Size: 3_000_000_000_000}, + {Id: 2, Size: 3_020_000_000_000}, + }, + }, + wantCapacity: 5_700_000_000_000, + wantUsage: 4_200_000_000_000, + }, + { + name: "plain filesystem reports statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 1_000_000_000_000, + DiskFreeBytes: 400_000_000_000, + VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 1, Size: 590_000_000_000}}, + }, + wantCapacity: 1_000_000_000_000, + wantUsage: 600_000_000_000, + }, + { + name: "ec shards are not added on top of statfs usage", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 100, + DiskTotalBytes: 1_000_000_000_000, + DiskFreeBytes: 300_000_000_000, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 3, EcIndexBits: 0b11, ShardSizes: []int64{100, 200}}, + }, + }, + wantCapacity: 1_000_000_000_000, + wantUsage: 700_000_000_000, + }, + { + name: "without statfs numbers falls back to logical size", + disk: &master_pb.DiskInfo{ + MaxVolumeCount: 10, + VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 4, Size: 30_000_000_000}}, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{300}}, + }, + }, + // 10 slots at the 30000 MiB default volume size limit. + wantCapacity: 10 * 30000 * 1024 * 1024, + wantUsage: 30_000_000_000 + 300, + }, + } { + t.Run(tc.name, func(t *testing.T) { + admin := newVolumeServersTestAdmin(t, volumeAccessTestTopology(&master_pb.DataNodeInfo{ + Id: "node-a", + Address: "127.0.0.1:8080", + GrpcPort: 18080, + DiskInfos: map[string]*master_pb.DiskInfo{"hdd": tc.disk}, + })) + + data, err := admin.GetClusterVolumeServers() + require.NoError(t, err) + require.Len(t, data.VolumeServers, 1) + server := data.VolumeServers[0] + require.Equal(t, tc.wantCapacity, server.DiskCapacity) + require.Equal(t, tc.wantUsage, server.DiskUsage) + require.LessOrEqual(t, server.DiskUsage, server.DiskCapacity, + "usage above capacity makes the advertised available space negative") + }) + } +}