wdclient: regression test for the elapsed caller-deadline lookup path (#12154)

* wdclient: cover a lapsed caller deadline that Err() has not observed yet

* wdclient: reuse isCanceledErr for the elapsed-deadline check

* wdclient: reinterpret only deadline errors as caller-gave-up

isCanceledErr also matches gRPC Canceled, which a filer can return for
its own reasons; treating it as an elapsed caller deadline hid the
failure from the filer's health record. The fallback now matches only
context and gRPC DeadlineExceeded, so filer-side cancellations keep
counting as filer failures.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-10-28 01:08:04 -07:00
1 parent 2b7c65048d
commit 7d823265e5
3 files changed
+29 -1

No files matched your search

+1 -1
View File
@@ -792,7 +792,7 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s
// surface its own DeadlineExceeded a tick before ctx.Err()
// becomes visible, so reinterpret only deadline errors this way
ctxErr := ctx.Err()
if ctxErr == nil && status.Code(err) == codes.DeadlineExceeded {
if ctxErr == nil && isDeadlineExceededErr(err) {
if dl, ok := ctx.Deadline(); ok && !dl.After(time.Now()) {
ctxErr = context.DeadlineExceeded
}
+17
View File
@@ -309,6 +309,14 @@ func startStallingFiler(t *testing.T) pb.ServerAddress {
return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
}
// lapsedDeadlineCtx mimics the moment after a caller's deadline has passed but
// before the runtime marks the context done.
type lapsedDeadlineCtx struct{ context.Context }
func (lapsedDeadlineCtx) Deadline() (time.Time, bool) { return time.Now().Add(-time.Second), true }
func (lapsedDeadlineCtx) Done() <-chan struct{} { return nil }
func (lapsedDeadlineCtx) Err() error { return nil }
// A caller giving up must not count as a filer failure, the filer's own timeout must.
func TestLookupVolumeIdsCountsOnlyFilerFailures(t *testing.T) {
tests := []struct {
@@ -338,6 +346,15 @@ func TestLookupVolumeIdsCountsOnlyFilerFailures(t *testing.T) {
wantErr: context.DeadlineExceeded,
wantFailures: 0,
},
{
name: "caller deadline already elapsed",
callerCtx: func() (context.Context, context.CancelFunc) {
return lapsedDeadlineCtx{context.Background()}, func() {}
},
grpcTimeout: 5 * time.Second,
wantErr: context.DeadlineExceeded,
wantFailures: 0,
},
{
name: "filer timeout",
callerCtx: func() (context.Context, context.CancelFunc) {
+11
View File
@@ -44,6 +44,17 @@ func isCanceledErr(err error) bool {
return false
}
func isDeadlineExceededErr(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
statusErr, ok := status.FromError(err)
return ok && statusErr.Code() == codes.DeadlineExceeded
}
// LookupVolumeIds queries the master for volume locations (fallback when cache misses).
// Returns partial results with aggregated errors for volumes that failed.
// Retries on codes.Unavailable (e.g. master warming up after restart) with backoff.