seaweedfs/weed/admin/dash/cluster_topology.go

package dash

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)

const dirStatusTimeout = 5 * time.Second

// GetClusterTopology returns the current cluster topology with caching
func (s *AdminServer) GetClusterTopology() (*ClusterTopology, error) {
	now := time.Now()
	if s.cachedTopology != nil && now.Sub(s.lastCacheUpdate) < s.cacheExpiration {
		return s.cachedTopology, nil
	}

	topology := &ClusterTopology{
		UpdatedAt: now,
	}

	// Use gRPC only
	err := s.getTopologyViaGRPC(topology)
	if err != nil {
		currentMaster := s.masterClient.GetMaster(context.Background())
		glog.Errorf("Failed to connect to master server %s: %v", currentMaster, err)
		return nil, fmt.Errorf("gRPC topology request failed: %w", err)
	}

	// Cache the result
	s.cachedTopology = topology
	s.lastCacheUpdate = now

	return topology, nil
}

// fetchPublicUrlMap queries the master's /dir/status HTTP endpoint and returns
// a map from data node ID (ip:port) to its PublicUrl.
func (s *AdminServer) fetchPublicUrlMap() map[string]string {
	currentMaster := s.masterClient.GetMaster(context.Background())
	if currentMaster == "" {
		return nil
	}

	ctx, cancel := context.WithTimeout(context.Background(), dirStatusTimeout)
	defer cancel()

	url := fmt.Sprintf("http://%s/dir/status", currentMaster.ToHttpAddress())
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		glog.V(1).Infof("Failed to build /dir/status request for %s: %v", currentMaster, err)
		return nil
	}
	resp, err := util_http.GetGlobalHttpClient().Do(req)
	if err != nil {
		glog.V(1).Infof("Failed to fetch /dir/status from %s: %v", currentMaster, err)
		return nil
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		glog.V(1).Infof("Non-OK response from /dir/status: %d", resp.StatusCode)
		return nil
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		glog.V(1).Infof("Failed to read /dir/status response body: %v", err)
		return nil
	}

	// Parse the JSON response to extract PublicUrl for each data node
	var status struct {
		Topology struct {
			DataCenters []struct {
				Racks []struct {
					DataNodes []struct {
						Url       string `json:"Url"`
						PublicUrl string `json:"PublicUrl"`
					} `json:"DataNodes"`
				} `json:"Racks"`
			} `json:"DataCenters"`
		} `json:"Topology"`
	}

	if err := json.Unmarshal(body, &status); err != nil {
		glog.V(1).Infof("Failed to parse /dir/status response: %v", err)
		return nil
	}

	publicUrls := make(map[string]string)
	for _, dc := range status.Topology.DataCenters {
		for _, rack := range dc.Racks {
			for _, dn := range rack.DataNodes {
				if dn.PublicUrl != "" {
					publicUrls[dn.Url] = dn.PublicUrl
				}
			}
		}
	}
	return publicUrls
}

// diskCapacityAndUsage returns the physical capacity of one disk together with
// the bytes in use on it, measured the same way. The bytes in use are the
// statfs numbers the volume server heartbeats, DiskTotalBytes-DiskFreeBytes,
// which is the same measurement CollectTierStats uses. The logical size of the
// volumes is not comparable with the physical capacity: a compressing
// filesystem (ZFS, Btrfs) stores fewer bytes than the volumes report, so
// counting logical bytes as usage made the admin UI show usage above capacity
// and derive a negative available space. Volume servers that do not report
// statfs numbers keep the previous behaviour: slot-based capacity and the
// logical bytes the disk holds.
func diskCapacityAndUsage(diskInfo *master_pb.DiskInfo, volumeSizeLimitMb uint64, logicalBytes int64) (capacity int64, usage int64) {
	if diskInfo.DiskTotalBytes > 0 {
		capacity = int64(diskInfo.DiskTotalBytes)
		if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes {
			usage = int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes)
		}
		return capacity, usage
	}
	return diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024, logicalBytes
}

// getTopologyViaGRPC gets topology using gRPC (original method)
func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
	// Fetch public URL mapping from master HTTP API
	// The gRPC DataNodeInfo does not include PublicUrl, so we supplement it.
	publicUrls := s.fetchPublicUrlMap()

	// Get cluster status from master
	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
		resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
		if err != nil {
			currentMaster := s.masterClient.GetMaster(context.Background())
			glog.Errorf("Failed to get volume list from master %s: %v", currentMaster, err)
			return err
		}

		if resp.TopologyInfo != nil {
			// Get volume size limit from response, default to 30GB if not set
			volumeSizeLimitMb := resp.VolumeSizeLimitMb
			if volumeSizeLimitMb == 0 {
				volumeSizeLimitMb = 30000
			}

			// Process gRPC response
			for _, dc := range resp.TopologyInfo.DataCenterInfos {
				dataCenter := DataCenter{
					ID:    dc.Id,
					Racks: []Rack{},
				}

				for _, rack := range dc.RackInfos {
					rackObj := Rack{
						ID:    rack.Id,
						Nodes: []VolumeServer{},
					}

					for _, node := range rack.DataNodeInfos {
						// Calculate totals from disk infos
						var totalVolumes int64
						var totalMaxVolumes int64
						var totalSize int64
						var remoteSize int64
						var diskCapacity int64
						var diskUsage int64

						for _, diskInfo := range node.DiskInfos {
							totalVolumes += diskInfo.VolumeCount
							totalMaxVolumes += diskInfo.MaxVolumeCount

							// A remote-tiered volume reports its cloud object's
							// size; keep those bytes out of the local disk usage
							// that is compared against diskCapacity.
							var diskLocalSize int64
							for _, volInfo := range diskInfo.VolumeInfos {
								if volInfo.RemoteStorageName != "" {
									remoteSize += int64(volInfo.Size)
								} else {
									diskLocalSize += int64(volInfo.Size)
								}
							}

							// ShardSizes is local to this node, so summing
							// across nodes gives the physical footprint.
							for _, ecShardInfo := range diskInfo.EcShardInfos {
								for _, shardSize := range ecShardInfo.ShardSizes {
									diskLocalSize += shardSize
								}
							}
							totalSize += diskLocalSize

							capacity, usage := diskCapacityAndUsage(diskInfo, volumeSizeLimitMb, diskLocalSize)
							diskCapacity += capacity
							diskUsage += usage
						}

						// Look up PublicUrl from master HTTP API
						// Use node.Address (ip:port) as the key, matching the Url field in /dir/status
						nodeAddr := node.Address
						if nodeAddr == "" {
							nodeAddr = node.Id
						}
						publicUrl := publicUrls[nodeAddr]
						if publicUrl == "" {
							publicUrl = nodeAddr
						}

						vs := VolumeServer{
							ID:            node.Id,
							Address:       node.Id,
							DataCenter:    dc.Id,
							Rack:          rack.Id,
							PublicURL:     publicUrl,
							Volumes:       int(totalVolumes),
							MaxVolumes:    int(totalMaxVolumes),
							DiskUsage:     diskUsage,
							DiskCapacity:  diskCapacity,
							LastHeartbeat: time.Now(),
							RemoteSize:    remoteSize,
						}

						rackObj.Nodes = append(rackObj.Nodes, vs)
						topology.VolumeServers = append(topology.VolumeServers, vs)
						topology.TotalVolumes += vs.Volumes
						// TotalSize is the logical data size, wherever the
						// bytes live, so remote-tiered volumes still count.
						topology.TotalSize += totalSize + remoteSize
						topology.DiskUsage += diskUsage
						topology.DiskCapacity += diskCapacity
					}

					dataCenter.Racks = append(dataCenter.Racks, rackObj)
				}

				topology.DataCenters = append(topology.DataCenters, dataCenter)
			}

			// Chunk counts come from the shared collection aggregation, which
			// nets out tombstones and counts a chunk once no matter how many
			// volume replicas or EC shard holders report it.
			topology.TotalChunks = totalCollectionFileCount(resp.TopologyInfo)

			topology.TierStats = CollectTierStats(resp.TopologyInfo, volumeSizeLimitMb)
		}

		return nil
	})

	return err
}

// InvalidateCache forces a refresh of cached data
func (s *AdminServer) InvalidateCache() {
	s.lastCacheUpdate = time.Now().Add(-s.cacheExpiration)
	s.cachedTopology = nil
	s.lastFilerUpdate = time.Now().Add(-s.filerCacheExpiration)
	s.cachedFilers = nil
	s.lastCollectionStatsUpdate = time.Now().Add(-s.collectionStatsCacheThreshold)
	s.collectionStatsCache = nil
}