package weed_server import ( "context" "fmt" "net/http" "net/http/httptest" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/sequence" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/topology" ) const assignTestVolumeSizeLimit = 1 << 30 // assignTestCluster is a leader master over one data center with one volume // server per rack, like a production cluster spreading "010" replicas. type assignTestCluster struct { ms *MasterServer nodes []*topology.DataNode rp *super_block.ReplicaPlacement nextVid uint32 } func newAssignTestCluster(t *testing.T, racks int, replication string) *assignTestCluster { t.Helper() rp, err := super_block.NewReplicaPlacementFromString(replication) require.NoError(t, err) c := &assignTestCluster{ms: newLeaderMaster(), rp: rp} // Room for the per-assign size estimate, which would fill newLeaderMaster's // tiny volumes on the first assign. c.ms.Topo = topology.NewTopology("test", sequence.NewMemorySequencer(), assignTestVolumeSizeLimit, 5, false) c.ms.Topo.RaftServer = leaderRaftServer{} dc := c.ms.Topo.GetOrCreateDataCenter("dc1") for i := 0; i < racks; i++ { c.nodes = append(c.nodes, dc.GetOrCreateRack(fmt.Sprintf("rack%d", i)). GetOrCreateDataNode("127.0.0.1", 8080+i, 18080+i, "127.0.0.1", fmt.Sprintf("dn%d", i), map[string]uint32{"": 1000})) } return c } // grow registers n new empty volumes, each with its copies on consecutive // racks, the way a finished volume grow lands in the topology. func (c *assignTestCluster) grow(collection string, n uint32) { for i := uint32(0); i < n; i++ { c.nextVid++ v := storage.VolumeInfo{ Id: needle.VolumeId(c.nextVid), Collection: collection, ReplicaPlacement: c.rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion(), } for r := 0; r < c.rp.GetCopyCount(); r++ { dn := c.nodes[(int(c.nextVid)+r)%len(c.nodes)] dn.AddOrUpdateVolume(v) c.ms.Topo.RegisterVolumeLayout(v, dn) } } } func (c *assignTestCluster) layout(collection string) *topology.VolumeLayout { return c.ms.Topo.GetVolumeLayout(collection, c.rp, needle.EMPTY_TTL, "") } func (c *assignTestCluster) httpAssign(t *testing.T, query string) { t.Helper() w := httptest.NewRecorder() c.ms.dirAssignHandler(w, httptest.NewRequest(http.MethodGet, "/dir/assign?"+query, nil)) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) } // A client's writableVolumeCount is what the periodic growth scan keeps a // layout topped up to. gRPC Assign has recorded it since the scan was added; // HTTP /dir/assign parsed it but dropped it, so HTTP-only writers never got // proactive growth. Both entry points must leave the layout in the same state. func TestDirAssignRecordsWritableVolumeCountLikeGrpcAssign(t *testing.T) { tests := []struct { name string counts []uint32 // writableVolumeCount per assign, 0 = not sent want uint32 }{ {name: "sent", counts: []uint32{30}, want: 30}, {name: "not sent", counts: []uint32{0}, want: 0}, {name: "later assign without it keeps the last value", counts: []uint32{30, 0}, want: 30}, {name: "latest value wins", counts: []uint32{30, 12}, want: 12}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { httpCluster := newAssignTestCluster(t, 1, "000") httpCluster.grow("c", 1) for _, n := range tt.counts { query := "collection=c&replication=000" if n > 0 { query += fmt.Sprintf("&writableVolumeCount=%d", n) } httpCluster.httpAssign(t, query) } grpcCluster := newAssignTestCluster(t, 1, "000") grpcCluster.grow("c", 1) for _, n := range tt.counts { _, err := grpcCluster.ms.Assign(context.Background(), &master_pb.AssignRequest{ Count: 1, Collection: "c", Replication: "000", WritableVolumeCount: n, }) require.NoError(t, err) } assert.Equal(t, tt.want, grpcCluster.layout("c").GetLastGrowCount(), "gRPC Assign") assert.Equal(t, tt.want, httpCluster.layout("c").GetLastGrowCount(), "HTTP /dir/assign") }) } } // An assign naming a placement the topology cannot serve must fail before it // records a growth target: the periodic scan drops the requested placement, so // the volumes it would keep growing could never serve the requester. func TestDirAssignRejectsUnknownPlacementBeforeGrowCount(t *testing.T) { c := newAssignTestCluster(t, 1, "000") c.grow("c", 1) vl := c.layout("c") // A rack and data node that exist, but in a different data center: the // requester named dc1, so these placements can never be served either. c.ms.Topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rackX"). GetOrCreateDataNode("127.0.0.2", 8090, 18090, "127.0.0.2", "dnX", map[string]uint32{"": 1000}) for _, query := range []string{ "collection=c&replication=000&writableVolumeCount=30&rack=missing", "collection=c&replication=000&writableVolumeCount=30&dataNode=missing", "collection=c&replication=000&writableVolumeCount=30&dataCenter=dc1&rack=rackX", "collection=c&replication=000&writableVolumeCount=30&dataCenter=dc1&dataNode=dnX", "collection=c&replication=000&writableVolumeCount=30&dataCenter=dc1&rack=rack0&dataNode=dnX", } { w := httptest.NewRecorder() c.ms.dirAssignHandler(w, httptest.NewRequest(http.MethodGet, "/dir/assign?"+query, nil)) assert.Equal(t, http.StatusBadRequest, w.Code, w.Body.String()) } assert.Zero(t, vl.GetLastGrowCount()) for _, req := range []*master_pb.AssignRequest{ {Count: 1, Collection: "c", Replication: "000", WritableVolumeCount: 30, Rack: "missing"}, {Count: 1, Collection: "c", Replication: "000", WritableVolumeCount: 30, DataCenter: "dc1", Rack: "rackX"}, {Count: 1, Collection: "c", Replication: "000", WritableVolumeCount: 30, DataCenter: "dc1", Rack: "rack0", DataNode: "dnX"}, } { _, err := c.ms.Assign(context.Background(), req) assert.Error(t, err) } assert.Zero(t, vl.GetLastGrowCount()) } // Replays the outage of an HTTP-only writer with writableVolumeCount=30 on a // "010" collection across 5 racks. Writes are spread by remaining capacity, so // the 30 volumes fill in lockstep. Before the fix the periodic scan only ever // saw lastGrowCount 0: it added the rack-aware step of 2 when everything was // crowded and nothing more while those 2 stayed uncrowded. When the old batch // went full, the 2 volumes took every write, filled before the next scan, and // assigns failed until an emergency grow landed. func TestPeriodicGrowthKeepsHTTPAssignLayoutWritable(t *testing.T) { const n = 30 c := newAssignTestCluster(t, 5, "010") c.grow("photo", n) vl := c.layout("photo") c.httpAssign(t, fmt.Sprintf("collection=photo&replication=010&writableVolumeCount=%d", n)) const limit = int64(assignTestVolumeSizeLimit) batch := vl.CloneWritableVolumes() writeTo := func(vids []needle.VolumeId, size int64) { for _, vid := range vids { vl.RecordAssign(vid, size) } } // scan runs the per-layout steps of ProcessGrowRequest, with a grow that // lands at once. scan := func() { writable, crowded := vl.GetWritableVolumeCount() count, _ := periodicGrowCount(vl.GetLastGrowCount(), writable, crowded) c.grow("photo", count) for _, plan := range vl.PlanRackAwareGrowth(c.ms.Topo.ListDCAndRacks(), vl.GetLastGrowCount(), volumeGrowStepCount) { c.grow("photo", plan.WritableVolumeCount) } writable, _ = vl.GetWritableVolumeCount() require.LessOrEqual(t, writable, 2*n, "growth must stay bounded by twice the requested writable count") } // The batch crosses the crowded threshold together, and stays crowded // over two scans. writeTo(batch, limit*95/100) _, crowded := vl.GetWritableVolumeCount() require.Equal(t, n, crowded) scan() scan() // The batch goes full; whatever the scans added is all that is left. writeTo(batch, limit) scan() writable, crowded := vl.GetWritableVolumeCount() assert.Zero(t, crowded) assert.GreaterOrEqual(t, writable, n, "after the batch went full the scan must restore the %d writable volumes the client asked for, not leave %d to take every write", n, writable) }