mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-04 11:07:35 +01:00
329ccb8bc0
* master: tell a warming-up Assign client when warm-up ends During warm-up the master refuses Assign with Unavailable and now attaches the remaining warm-up as a gRPC RetryInfo detail. RetryWithBackoff may shorten a backoff step to that hint plus 50 ms, never lengthen it, and only for a valid, positive hint, so a writer retries right after warm-up ends instead of up to one backoff step later. The warm-up itself is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: jitter the retry-hint slack and hint LookupVolume too Per review: the slack added to a server hint is now random in [50ms, 300ms], so clients refused at the same moment do not all retry in one instant. The warm-up refusal of LookupVolume carries the same RetryInfo hint as Assign; the masterclient's lookup retry already goes through RetryWithBackoff, so it lands at warm-up end too. Tests: the slow guard tests are replaced by one sleep-free table test of the step logic (zero, negative, huge, malformed, valid and wrapped hints, a hint longer than the step, none). One timing test of the shortened step stays, plus a jitter-bounds test and a LookupVolume hint test. The no-hint control test, which was timing-sensitive, is dropped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: name the shared warm-up test and file the lookup test by topic Rename the no-remaining test to TestWarmingUpErrorWithoutRemainingHasNoHint, since the helper now serves Assign and LookupVolume, and move the LookupVolume hint test next to the other LookupVolume warm-up tests. Tests only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * util: read the retry hint through ServerStatus Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: bound the warm-up hint in tests by the remaining warm-up around the RPC Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: leave more scheduler margin in the retry-hint timing tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * util, master: trim retry-hint comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: widen the retry-hint timing bounds to just under the unhinted wait Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: require a refused warm-up call before the hinted assign retry Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: check the hinted retry wait through a test timer, not the wall clock Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: move the assign warm-up end to 2.6s for more hinted-retry headroom Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: require the hinted retry to wait out its timer Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
72 lines
2.6 KiB
Go
72 lines
2.6 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// A leader that already knew volumes at the leader change is warming up until
|
|
// every volume server has had time to report again.
|
|
func newWarmingUpMaster(t *testing.T) *MasterServer {
|
|
t.Helper()
|
|
ms := newLeaderMaster()
|
|
node := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "node1", map[string]uint32{"": 10})
|
|
ms.Topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{Id: 7, Size: 100}}, node)
|
|
ms.Topo.SetLastLeaderChangeTime(time.Now())
|
|
if !ms.Topo.IsWarmingUp() {
|
|
t.Fatal("precondition: master is not warming up")
|
|
}
|
|
return ms
|
|
}
|
|
|
|
func TestLookupVolumeDuringWarmupRefusesPartialNotFound(t *testing.T) {
|
|
ms := newWarmingUpMaster(t)
|
|
_, err := ms.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: []string{"7", "8"}})
|
|
if status.Code(err) != codes.Unavailable {
|
|
t.Fatalf("partial not-found during warmup = %v, want Unavailable", err)
|
|
}
|
|
}
|
|
|
|
func TestLookupVolumeDuringWarmupAnswersFullyKnownBatch(t *testing.T) {
|
|
ms := newWarmingUpMaster(t)
|
|
resp, err := ms.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: []string{"7"}})
|
|
if err != nil {
|
|
t.Fatalf("known volume during warmup: %v", err)
|
|
}
|
|
if len(resp.VolumeIdLocations) != 1 || len(resp.VolumeIdLocations[0].Locations) != 1 || resp.VolumeIdLocations[0].Error != "" {
|
|
t.Fatalf("known volume answer = %+v", resp.VolumeIdLocations)
|
|
}
|
|
}
|
|
|
|
func TestLookupVolumeAfterWarmupReportsNotFoundPerVolume(t *testing.T) {
|
|
ms := newWarmingUpMaster(t)
|
|
ms.Topo.SetLastLeaderChangeTime(time.Now().Add(-time.Hour))
|
|
resp, err := ms.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: []string{"7", "8"}})
|
|
if err != nil {
|
|
t.Fatalf("lookup after warmup: %v", err)
|
|
}
|
|
if len(resp.VolumeIdLocations) != 2 {
|
|
t.Fatalf("answers = %+v, want two", resp.VolumeIdLocations)
|
|
}
|
|
if len(resp.VolumeIdLocations[0].Locations) != 1 || resp.VolumeIdLocations[0].Error != "" {
|
|
t.Fatalf("known volume answer = %+v", resp.VolumeIdLocations[0])
|
|
}
|
|
if len(resp.VolumeIdLocations[1].Locations) != 0 || resp.VolumeIdLocations[1].Error == "" {
|
|
t.Fatalf("absent volume answer = %+v", resp.VolumeIdLocations[1])
|
|
}
|
|
}
|
|
|
|
func TestLookupVolumeDuringWarmupCarriesRemainingWarmup(t *testing.T) {
|
|
ms := newWarmingUpMaster(t)
|
|
assertWarmupHint(t, ms, func() error {
|
|
_, err := ms.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: []string{"7", "8"}})
|
|
return err
|
|
})
|
|
}
|