mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
master: export writable volumes per disk and reset leader-only layout gauges on followers (#12751)
* master: export writable volumes per disk for each volume layout The layout gauges only report how many volumes of a collection take writes, not where they sit, so two writable volumes sharing one disk cannot be seen or alerted on. The leader now exports writable_volumes_per_disk with the layout labels plus data center, rack, node and disk id, counting the replicas of the layout's writable volumes on each disk (state=writable) and those of them that are crowded (state=crowded). Series for disks that no longer hold a writable volume of the layout, and for layouts that are gone, are deleted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: reset the leader-only layout gauges on followers The layout writable, crowded, under-replicated and per-disk gauges are only set by the leader, so a master that lost leadership kept exporting the values of its last term indefinitely, and dashboards that aggregate over masters could show numbers days old. A master that is not the leader now clears them on each periodic pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart disks holding several writable volumes of a collection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: keep disks of different clusters apart in the per-disk panel Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1fd5008d63
commit
7b4bdb77bb
6 files changed
+395
-1
No files matched your search
@@ -2302,6 +2302,113 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Writable Volumes per Disk",
|
||||
"description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.",
|
||||
"type": "timeseries",
|
||||
"id": 262,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{collection}} disks with 2+"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "{{collection}} max on one disk"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -62,9 +63,60 @@ func periodicGrowCount(lastGrowCount uint32, writable, crowded int) (count uint3
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// writableDiskSeries keeps the label values of the per-disk series last set
|
||||
// for each layout, so disks that stop holding its writable volumes drop out.
|
||||
type writableDiskSeries map[*topology.VolumeLayout]map[string][]string
|
||||
|
||||
func (s writableDiskSeries) update(vl *topology.VolumeLayout, layout ...string) {
|
||||
writable, crowded := vl.WritableVolumesPerDisk()
|
||||
current := make(map[string][]string, 2*len(writable))
|
||||
for disk, count := range writable {
|
||||
for _, state := range []struct {
|
||||
name string
|
||||
count int
|
||||
}{{"writable", count}, {"crowded", crowded[disk]}} {
|
||||
labels := append(append(make([]string, 0, len(layout)+5), layout...),
|
||||
disk.DataCenter, disk.Rack, disk.Node, strconv.FormatUint(uint64(disk.DiskId), 10), state.name)
|
||||
stats.MasterWritableVolumesPerDisk.WithLabelValues(labels...).Set(float64(state.count))
|
||||
current[strings.Join(labels, "\x00")] = labels
|
||||
}
|
||||
}
|
||||
for key, labels := range s[vl] {
|
||||
if _, ok := current[key]; !ok {
|
||||
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
|
||||
}
|
||||
}
|
||||
s[vl] = current
|
||||
}
|
||||
|
||||
func (s writableDiskSeries) retain(layouts []*topology.VolumeLayoutCollection) {
|
||||
listed := make(map[*topology.VolumeLayout]bool, len(layouts))
|
||||
for _, vlc := range layouts {
|
||||
listed[vlc.VolumeLayout] = true
|
||||
}
|
||||
for vl, series := range s {
|
||||
if listed[vl] {
|
||||
continue
|
||||
}
|
||||
for _, labels := range series {
|
||||
stats.MasterWritableVolumesPerDisk.DeleteLabelValues(labels...)
|
||||
}
|
||||
delete(s, vl)
|
||||
}
|
||||
}
|
||||
|
||||
func resetLeaderLayoutGauges(diskSeries writableDiskSeries) {
|
||||
stats.MasterVolumeLayoutWritable.Reset()
|
||||
stats.MasterVolumeLayoutCrowded.Reset()
|
||||
stats.MasterUnderReplicatedVolumes.Reset()
|
||||
stats.MasterWritableVolumesPerDisk.Reset()
|
||||
clear(diskSeries)
|
||||
}
|
||||
|
||||
func (ms *MasterServer) ProcessGrowRequest() {
|
||||
go func() {
|
||||
ctx := context.Background()
|
||||
diskSeries := make(writableDiskSeries)
|
||||
firstRun := true
|
||||
for {
|
||||
if firstRun {
|
||||
@@ -73,10 +125,13 @@ func (ms *MasterServer) ProcessGrowRequest() {
|
||||
time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second)
|
||||
}
|
||||
if !ms.Topo.IsLeader() {
|
||||
resetLeaderLayoutGauges(diskSeries)
|
||||
continue
|
||||
}
|
||||
dcs := ms.Topo.ListDCAndRacks()
|
||||
for _, vlc := range ms.Topo.ListVolumeLayoutCollections() {
|
||||
layouts := ms.Topo.ListVolumeLayoutCollections()
|
||||
diskSeries.retain(layouts)
|
||||
for _, vlc := range layouts {
|
||||
var err error
|
||||
vl := vlc.VolumeLayout
|
||||
lastGrowCount := vl.GetLastGrowCount()
|
||||
@@ -86,6 +141,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
|
||||
writable, crowded := vl.GetWritableVolumeCount()
|
||||
vgr := vlc.ToVolumeGrowRequest()
|
||||
underReplicated := vl.CountUnderReplicatedVolumes()
|
||||
diskSeries.update(vl, vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl)
|
||||
stats.MasterVolumeLayoutWritable.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(writable))
|
||||
stats.MasterVolumeLayoutCrowded.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(crowded))
|
||||
stats.MasterUnderReplicatedVolumes.WithLabelValues(vlc.Collection, vgr.DiskType, vgr.Replication, vgr.Ttl).Set(float64(underReplicated))
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology"
|
||||
)
|
||||
|
||||
func writableVolumesPerDiskSeries(t *testing.T, collection string) map[string]float64 {
|
||||
t.Helper()
|
||||
ch := make(chan prometheus.Metric, 64)
|
||||
go func() {
|
||||
stats.MasterWritableVolumesPerDisk.Collect(ch)
|
||||
close(ch)
|
||||
}()
|
||||
got := make(map[string]float64)
|
||||
for m := range ch {
|
||||
var metric dto.Metric
|
||||
if err := m.Write(&metric); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
labels := make(map[string]string)
|
||||
for _, l := range metric.GetLabel() {
|
||||
labels[l.GetName()] = l.GetValue()
|
||||
}
|
||||
if labels["collection"] == collection {
|
||||
got[labels["node"]+"/"+labels["disk_id"]+"/"+labels["state"]] = metric.GetGauge().GetValue()
|
||||
}
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestWritableDiskSeriesDropsDisksWithoutWritableVolumes(t *testing.T) {
|
||||
const collection = "perdisk"
|
||||
topo := topology.NewTopology("perdisk", nil, 10000, 5, false)
|
||||
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: collection, Version: 3, DiskId: 0},
|
||||
{Id: 2, Collection: collection, Version: 3, DiskId: 0},
|
||||
{Id: 3, Collection: collection, Version: 3, DiskId: 1},
|
||||
}, dn)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||
vl.SetVolumeCrowded(needle.VolumeId(3))
|
||||
node := string(dn.Id())
|
||||
|
||||
series := make(writableDiskSeries)
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
||||
node + "/0/writable": 2, node + "/0/crowded": 0,
|
||||
node + "/1/writable": 1, node + "/1/crowded": 1,
|
||||
})
|
||||
|
||||
vl.SetVolumeReadOnly(dn, needle.VolumeId(3))
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{
|
||||
node + "/0/writable": 2, node + "/0/crowded": 0,
|
||||
})
|
||||
|
||||
series.retain(nil)
|
||||
assertSeries(t, writableVolumesPerDiskSeries(t, collection), map[string]float64{})
|
||||
if len(series) != 0 {
|
||||
t.Errorf("series still tracks %d layouts", len(series))
|
||||
}
|
||||
}
|
||||
|
||||
func assertSeries(t *testing.T, got, want map[string]float64) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("series = %v, want %v", got, want)
|
||||
}
|
||||
for k, v := range want {
|
||||
if g, ok := got[k]; !ok || g != v {
|
||||
t.Errorf("series[%s] = %v (present %v), want %v", k, g, ok, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetLeaderLayoutGaugesClearsFollowerValues(t *testing.T) {
|
||||
const collection = "follower"
|
||||
topo := topology.NewTopology("follower", nil, 10000, 5, false)
|
||||
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: collection, Version: 3},
|
||||
}, dn)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
vl := topo.GetVolumeLayout(collection, rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||
|
||||
series := make(writableDiskSeries)
|
||||
series.update(vl, collection, "hdd", "000", "")
|
||||
stats.MasterVolumeLayoutWritable.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
stats.MasterVolumeLayoutCrowded.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
stats.MasterUnderReplicatedVolumes.WithLabelValues(collection, "hdd", "000", "").Set(1)
|
||||
|
||||
resetLeaderLayoutGauges(series)
|
||||
|
||||
for name, gauge := range map[string]*prometheus.GaugeVec{
|
||||
"volume_layout_writable": stats.MasterVolumeLayoutWritable,
|
||||
"volume_layout_crowded": stats.MasterVolumeLayoutCrowded,
|
||||
"under_replicated_volumes": stats.MasterUnderReplicatedVolumes,
|
||||
"writable_volumes_per_disk": stats.MasterWritableVolumesPerDisk,
|
||||
} {
|
||||
if n := testutil.CollectAndCount(gauge); n != 0 {
|
||||
t.Errorf("%s still exports %d series after the reset", name, n)
|
||||
}
|
||||
}
|
||||
if len(series) != 0 {
|
||||
t.Errorf("series still tracks %d layouts", len(series))
|
||||
}
|
||||
}
|
||||
@@ -159,6 +159,14 @@ var (
|
||||
Help: "Current number of volumes that do not have enough replicas per collection/layout. 0 = healthy.",
|
||||
}, []string{"collection", "disk", "rp", "ttl"})
|
||||
|
||||
MasterWritableVolumesPerDisk = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemMaster,
|
||||
Name: "writable_volumes_per_disk",
|
||||
Help: "Number of writable volumes of a volume layout with a replica on the disk; state=crowded counts those past the grow threshold.",
|
||||
}, []string{"collection", "disk", "rp", "ttl", "data_center", "rack", "node", "disk_id", "state"})
|
||||
|
||||
// MasterVolumeCreationCounter counts volume creation (growth) operations by result (success, failure).
|
||||
// Volume growth is orchestrated by the master, so this metric is exported by the master process.
|
||||
MasterVolumeCreationCounter = prometheus.NewCounterVec(
|
||||
@@ -1121,6 +1129,7 @@ func init() {
|
||||
Gather.MustRegister(VolumeServerECRebuildCounter)
|
||||
Gather.MustRegister(VolumeServerECReconstructedIntervalCounter)
|
||||
Gather.MustRegister(MasterUnderReplicatedVolumes)
|
||||
Gather.MustRegister(MasterWritableVolumesPerDisk)
|
||||
Gather.MustRegister(MasterVolumeCreationCounter)
|
||||
|
||||
Gather.MustRegister(S3RequestCounter)
|
||||
@@ -1264,6 +1273,7 @@ func DeleteCollectionMetrics(collection string) {
|
||||
c += MasterVolumeLayoutWritable.DeletePartialMatch(labels)
|
||||
c += MasterVolumeLayoutCrowded.DeletePartialMatch(labels)
|
||||
c += MasterUnderReplicatedVolumes.DeletePartialMatch(labels)
|
||||
c += MasterWritableVolumesPerDisk.DeletePartialMatch(labels)
|
||||
c += VolumeServerDiskSizeGauge.DeletePartialMatch(labels)
|
||||
c += VolumeServerVolumeGauge.DeletePartialMatch(labels)
|
||||
c += VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(labels)
|
||||
|
||||
@@ -1140,6 +1140,51 @@ func (vl *VolumeLayout) CountUnderReplicatedVolumes() int {
|
||||
return count
|
||||
}
|
||||
|
||||
type WritableDisk struct {
|
||||
DataCenter string
|
||||
Rack string
|
||||
Node string
|
||||
DiskId uint32
|
||||
}
|
||||
|
||||
// WritableVolumesPerDisk counts, per disk, the replicas of this layout's
|
||||
// writable volumes, and how many of those are crowded.
|
||||
func (vl *VolumeLayout) WritableVolumesPerDisk() (writable, crowded map[WritableDisk]int) {
|
||||
type replicas struct {
|
||||
vid needle.VolumeId
|
||||
nodes []*DataNode
|
||||
crowded bool
|
||||
}
|
||||
vl.accessLock.RLock()
|
||||
snapshot := make([]replicas, 0, len(vl.writables))
|
||||
for _, vid := range vl.writables {
|
||||
if locations, ok := vl.vid2location[vid]; ok {
|
||||
_, isCrowded := vl.crowded[vid]
|
||||
snapshot = append(snapshot, replicas{vid, append([]*DataNode(nil), locations.list...), isCrowded})
|
||||
}
|
||||
}
|
||||
vl.accessLock.RUnlock()
|
||||
|
||||
writable, crowded = make(map[WritableDisk]int), make(map[WritableDisk]int)
|
||||
for _, r := range snapshot {
|
||||
for _, dn := range r.nodes {
|
||||
v, err := dn.GetVolumesById(r.vid)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
disk := WritableDisk{DataCenter: dn.GetDataCenterId(), Node: string(dn.Id()), DiskId: v.DiskId}
|
||||
if rack := dn.Parent(); rack != nil {
|
||||
disk.Rack = string(rack.Id())
|
||||
}
|
||||
writable[disk]++
|
||||
if r.crowded {
|
||||
crowded[disk]++
|
||||
}
|
||||
}
|
||||
}
|
||||
return writable, crowded
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) removeFromWritable(vid needle.VolumeId) bool {
|
||||
if _, ok := vl.writableMembers[vid]; !ok {
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func TestWritableVolumesPerDisk(t *testing.T) {
|
||||
topo := NewTopology("perdisk", nil, 10000, 5, false)
|
||||
dn1 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
dn2 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack2").
|
||||
GetOrCreateDataNode("127.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100})
|
||||
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
|
||||
{Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
|
||||
{Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true},
|
||||
}, dn1)
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
||||
{Id: 1, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 0},
|
||||
{Id: 2, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1},
|
||||
{Id: 3, Collection: "c", Version: 3, ReplicaPlacement: 10, DiskId: 1, ReadOnly: true},
|
||||
}, dn2)
|
||||
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("010")
|
||||
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||
vl.SetVolumeCrowded(needle.VolumeId(2))
|
||||
|
||||
writable, crowded := vl.WritableVolumesPerDisk()
|
||||
d1 := WritableDisk{DataCenter: "dc1", Rack: "rack1", Node: string(dn1.Id()), DiskId: 0}
|
||||
d20 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 0}
|
||||
d21 := WritableDisk{DataCenter: "dc1", Rack: "rack2", Node: string(dn2.Id()), DiskId: 1}
|
||||
wantWritable := map[WritableDisk]int{d1: 2, d20: 1, d21: 1}
|
||||
wantCrowded := map[WritableDisk]int{d1: 1, d21: 1}
|
||||
if len(writable) != len(wantWritable) {
|
||||
t.Fatalf("writable = %v, want %v", writable, wantWritable)
|
||||
}
|
||||
for d, n := range wantWritable {
|
||||
if writable[d] != n {
|
||||
t.Errorf("writable[%+v] = %d, want %d", d, writable[d], n)
|
||||
}
|
||||
}
|
||||
if len(crowded) != len(wantCrowded) {
|
||||
t.Fatalf("crowded = %v, want %v", crowded, wantCrowded)
|
||||
}
|
||||
for d, n := range wantCrowded {
|
||||
if crowded[d] != n {
|
||||
t.Errorf("crowded[%+v] = %d, want %d", d, crowded[d], n)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user