mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-04 11:07:35 +01:00
* master: tell a warming-up Assign client when warm-up ends During warm-up the master refuses Assign with Unavailable and now attaches the remaining warm-up as a gRPC RetryInfo detail. RetryWithBackoff may shorten a backoff step to that hint plus 50 ms, never lengthen it, and only for a valid, positive hint, so a writer retries right after warm-up ends instead of up to one backoff step later. The warm-up itself is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: jitter the retry-hint slack and hint LookupVolume too Per review: the slack added to a server hint is now random in [50ms, 300ms], so clients refused at the same moment do not all retry in one instant. The warm-up refusal of LookupVolume carries the same RetryInfo hint as Assign; the masterclient's lookup retry already goes through RetryWithBackoff, so it lands at warm-up end too. Tests: the slow guard tests are replaced by one sleep-free table test of the step logic (zero, negative, huge, malformed, valid and wrapped hints, a hint longer than the step, none). One timing test of the shortened step stays, plus a jitter-bounds test and a LookupVolume hint test. The no-hint control test, which was timing-sensitive, is dropped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: name the shared warm-up test and file the lookup test by topic Rename the no-remaining test to TestWarmingUpErrorWithoutRemainingHasNoHint, since the helper now serves Assign and LookupVolume, and move the LookupVolume hint test next to the other LookupVolume warm-up tests. Tests only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * util: read the retry hint through ServerStatus Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * master: bound the warm-up hint in tests by the remaining warm-up around the RPC Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: leave more scheduler margin in the retry-hint timing tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * util, master: trim retry-hint comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: widen the retry-hint timing bounds to just under the unhinted wait Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: require a refused warm-up call before the hinted assign retry Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: check the hinted retry wait through a test timer, not the wall clock Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: move the assign warm-up end to 2.6s for more hinted-retry headroom Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: require the hinted retry to wait out its timer Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
50 lines
1.5 KiB
Go
50 lines
1.5 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// assertWarmupHint calls refuse during warm-up and checks it is refused with
|
|
// Unavailable and a RetryInfo delay within the remaining warm-up measured
|
|
// around the call.
|
|
func assertWarmupHint(t *testing.T, ms *MasterServer, refuse func() error) {
|
|
t.Helper()
|
|
before := ms.Topo.RemainingWarmupDuration()
|
|
err := refuse()
|
|
after := ms.Topo.RemainingWarmupDuration()
|
|
st, _ := status.FromError(err)
|
|
var hint *errdetails.RetryInfo
|
|
for _, d := range st.Details() {
|
|
if ri, ok := d.(*errdetails.RetryInfo); ok {
|
|
hint = ri
|
|
}
|
|
}
|
|
if st.Code() != codes.Unavailable || hint == nil {
|
|
t.Fatalf("refused with %v and details %v, want Unavailable with RetryInfo", err, st.Details())
|
|
}
|
|
if got := hint.GetRetryDelay().AsDuration(); got <= 0 || got > before || got < after {
|
|
t.Fatalf("RetryInfo delay = %v, want within [%v, %v]", got, after, before)
|
|
}
|
|
}
|
|
|
|
func TestAssignDuringWarmupCarriesRemainingWarmup(t *testing.T) {
|
|
ms := newWarmingUpMaster(t)
|
|
assertWarmupHint(t, ms, func() error {
|
|
_, err := ms.Assign(context.Background(), &master_pb.AssignRequest{Count: 1})
|
|
return err
|
|
})
|
|
}
|
|
|
|
func TestWarmingUpErrorWithoutRemainingHasNoHint(t *testing.T) {
|
|
st, _ := status.FromError(warmingUpError(0))
|
|
if st.Code() != codes.Unavailable || len(st.Details()) != 0 {
|
|
t.Fatalf("got %v with details %v, want plain Unavailable", st.Code(), st.Details())
|
|
}
|
|
}
|