From 972f85159ef2fca65b4630e65d633bd3ae06ed8d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 19 Nov 2026 12:44:10 -0800 Subject: [PATCH] util/http: probe dead replicas in the background and key no-answer dials by machine (#12776) * util/http: probe an unanswering server in the background, not on a read Once the retry interval passed, the first read put the dead server first and waited out the full dial timeout before trying a live replica. Keep it last and dial it from a background probe with a short timeout instead. Co-Authored-By: Claude Opus 5.5 * util/http: try every server of a machine that does not answer last Unreachability was kept per host:port, so a powered-off node running one volume server per disk cost one stalled read per server before each was known dead. A dial that times out or finds no route now marks the whole machine; a refused connection still marks only that port and shows the machine is up. Co-Authored-By: Claude Opus 5.5 * util/http: probe an unanswering server with an HTTP request A bare TCP connect cleared servers that accept connections but never send response headers, and could not dial an address without an explicit port. Probe with a HEAD request through the global client instead, and have the tests wait for the probe to settle. Co-Authored-By: Claude Opus 5.5 * util/http: keep a timed-out port last after a sibling port answers A dial timeout stored only the machine verdict, so a success on another port of that machine cleared it and the port that timed out went back to the front. Record the port as well as the machine. Co-Authored-By: Claude Opus 5.5 * util/http: classify refused and unroutable dials with Winsock errnos on Windows Windows reports these as WSAECONNREFUSED, WSAEHOSTUNREACH and friends, which errors.Is does not match against the syscall constants. Test the refusal with a real dial to a closed port. Co-Authored-By: Claude Opus 5.5 * util/http: wait for background probes through a WaitGroup in tests A refused probe can finish before the test reads the claimed record, so inferring completion from the record's timestamp could wait out the deadline on a probe that already settled. Co-Authored-By: Claude Opus 5.5 * util/http: key machine verdicts apart from hosts without a port A host without an explicit port is also its machine name, so a sibling port answering cleared the timed-out host along with the machine verdict. Co-Authored-By: Claude Opus 5.5 * util/http: group a bracketed port-less IPv6 host with its machine net.SplitHostPort rejects "[2001:db8::9]", so the machine key kept the brackets and missed the verdict stored for "[2001:db8::9]:9501". Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/util/http/http_global_client_util.go | 6 +- weed/util/http/reachability.go | 123 ++++++++++++--- weed/util/http/reachability_errno.go | 16 ++ weed/util/http/reachability_errno_windows.go | 15 ++ weed/util/http/reachability_test.go | 154 ++++++++++++++++--- 5 files changed, 269 insertions(+), 45 deletions(-) create mode 100644 weed/util/http/reachability_errno.go create mode 100644 weed/util/http/reachability_errno_windows.go diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.go index c21e3df59e..9a4848811d 100644 --- a/weed/util/http/http_global_client_util.go +++ b/weed/util/http/http_global_client_util.go @@ -112,7 +112,7 @@ func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error response, err := GetGlobalHttpClient().Do(request) if err != nil { if ctx.Err() == nil { - recordUnreachable(request.URL.Host) + recordUnreachable(request.URL.Host, err) } return nil, true, err } @@ -401,7 +401,7 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, r, err := GetGlobalHttpClient().Do(req) if err != nil { if ctx.Err() == nil { - recordUnreachable(req.URL.Host) + recordUnreachable(req.URL.Host, err) } return true, err } @@ -801,7 +801,7 @@ func readUrlDirectToBuffer(ctx context.Context, fileUrl, jwt string, buffer []by r, err := GetGlobalHttpClient().Do(req) if err != nil { if ctx.Err() == nil { - recordUnreachable(req.URL.Host) + recordUnreachable(req.URL.Host, err) } return 0, true, err } diff --git a/weed/util/http/reachability.go b/weed/util/http/reachability.go index ad277b3eff..d97200183f 100644 --- a/weed/util/http/reachability.go +++ b/weed/util/http/reachability.go @@ -1,54 +1,119 @@ package http import ( + "context" + "errors" + "net" + "net/http" "net/url" + "strings" "sync" "time" ) // unreachableRetryInterval is how long a server that failed to answer is tried -// last before a read probes it again. +// last before it is probed again. const unreachableRetryInterval = 30 * time.Second -// unreachable maps a host to when it last failed to answer a request. A dead -// replica sits in the location list of every volume it held, so this is kept -// per host rather than per volume. -var unreachable sync.Map +// reachabilityProbeTimeout bounds the background request that checks whether +// an unanswering server is back. +const reachabilityProbeTimeout = 5 * time.Second -func recordUnreachable(host string) { - unreachable.Store(host, time.Now()) +type unanswered struct { + at time.Time + addr string } +// unreachable maps a host, and a machine whose dial got no answer, to its last +// failure. A dead replica sits in the location list of every volume it held, +// so this is kept per host rather than per volume. +var unreachable sync.Map + +// machineKey keys a machine apart from a host without an explicit port. +type machineKey string + +var probes sync.WaitGroup + +// recordUnreachable keys a failure by machine when the dial never reached it, +// so every server on a powered-off node is tried last after one failure. +func recordUnreachable(host string, err error) { + failure := unanswered{at: time.Now(), addr: host} + machine := machineOf(host) + unreachable.Store(host, failure) + if hostDown(err) { + unreachable.Store(machine, failure) + } else if connRefused(err) { + unreachable.Delete(machine) + } +} + +// recordReachable clears the machine verdict too, but a sibling port that +// failed on its own stays last until it answers. func recordReachable(host string) { unreachable.Delete(host) + unreachable.Delete(machineOf(host)) +} + +// hostDown reports whether a dial got no answer from the machine at all, unlike +// a refused connection, which shows the machine is up. +func hostDown(err error) bool { + var opErr *net.OpError + if !errors.As(err, &opErr) || opErr.Op != "dial" { + return false + } + return opErr.Timeout() || noRoute(err) } // ReachableFirst orders urls so that hosts which recently failed to answer // come last, keeping the order within each group. Once the retry interval has -// passed, the first read to ask claims the probe and tries that host first; -// the others keep it last until the probe settles. A read probes at most one -// host, so a claim is always followed by an attempt and several expired hosts -// are probed by successive reads. +// passed, such a host is probed in the background rather than by a read, so a +// dead replica never costs a read the dial timeout again. func ReachableFirst(urls []string) []string { - var probe, reachable, unanswered []string + var reachable, failed []string for _, u := range urls { - host := hostOf(u) - failedAt, failed := unreachable.Load(host) - switch { - case !failed: + if isUnanswering(hostOf(u)) { + failed = append(failed, u) + } else { reachable = append(reachable, u) - case time.Since(failedAt.(time.Time)) < unreachableRetryInterval: - unanswered = append(unanswered, u) - case probe == nil && unreachable.CompareAndSwap(host, failedAt, time.Now()): - probe = append(probe, u) - default: - unanswered = append(unanswered, u) } } - if probe == nil && unanswered == nil { + if failed == nil { return urls } - return append(append(probe, reachable...), unanswered...) + return append(reachable, failed...) +} + +func isUnanswering(host string) bool { + for _, key := range [2]any{host, machineOf(host)} { + v, found := unreachable.Load(key) + if !found { + continue + } + failure := v.(unanswered) + if time.Since(failure.at) >= unreachableRetryInterval && + unreachable.CompareAndSwap(key, failure, unanswered{at: time.Now(), addr: failure.addr}) { + probes.Go(func() { probeReachability(failure.addr) }) + } + return true + } + return false +} + +// probeReachability asks for response headers, not just a connection, so a +// server that accepts but never answers stays last. +func probeReachability(addr string) { + ctx, cancel := context.WithTimeout(context.Background(), reachabilityProbeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+addr+"/status", nil) + if err == nil { + var resp *http.Response + if resp, err = GetGlobalHttpClient().Do(req); err == nil { + resp.Body.Close() + recordReachable(addr) + return + } + } + recordUnreachable(addr, err) } func hostOf(rawUrl string) string { @@ -58,3 +123,13 @@ func hostOf(rawUrl string) string { } return parsed.Host } + +func machineOf(host string) machineKey { + if machine, _, err := net.SplitHostPort(host); err == nil { + return machineKey(machine) + } + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = host[1 : len(host)-1] + } + return machineKey(host) +} diff --git a/weed/util/http/reachability_errno.go b/weed/util/http/reachability_errno.go new file mode 100644 index 0000000000..8856557807 --- /dev/null +++ b/weed/util/http/reachability_errno.go @@ -0,0 +1,16 @@ +//go:build !windows + +package http + +import ( + "errors" + "syscall" +) + +func connRefused(err error) bool { + return errors.Is(err, syscall.ECONNREFUSED) +} + +func noRoute(err error) bool { + return errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTDOWN) +} diff --git a/weed/util/http/reachability_errno_windows.go b/weed/util/http/reachability_errno_windows.go new file mode 100644 index 0000000000..18cee5b260 --- /dev/null +++ b/weed/util/http/reachability_errno_windows.go @@ -0,0 +1,15 @@ +package http + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +func connRefused(err error) bool { + return errors.Is(err, windows.WSAECONNREFUSED) +} + +func noRoute(err error) bool { + return errors.Is(err, windows.WSAEHOSTUNREACH) || errors.Is(err, windows.WSAENETUNREACH) || errors.Is(err, windows.WSAEHOSTDOWN) +} diff --git a/weed/util/http/reachability_test.go b/weed/util/http/reachability_test.go index 4901ede9bb..ad7c5149d9 100644 --- a/weed/util/http/reachability_test.go +++ b/weed/util/http/reachability_test.go @@ -2,11 +2,14 @@ package http import ( "context" + "io" + "net" "net/http" "net/http/httptest" + "net/url" + "os" "sync/atomic" "testing" - "time" ) func forgetUnreachable(t *testing.T) { @@ -18,7 +21,10 @@ func forgetUnreachable(t *testing.T) { }) } forget() - t.Cleanup(forget) + t.Cleanup(func() { + probes.Wait() + forget() + }) } func assertOrder(t *testing.T, got []string, want ...string) { @@ -38,34 +44,146 @@ func TestReachableFirstTriesUnansweringHostLast(t *testing.T) { urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"} assertOrder(t, ReachableFirst(urls), urls...) - recordUnreachable("a:8080") + recordUnreachable("a:8080", io.EOF) assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) recordReachable("a:8080") assertOrder(t, ReachableFirst(urls), urls...) } -func TestReachableFirstProbesOnceAfterRetryInterval(t *testing.T) { - forgetUnreachable(t) - urls := []string{"http://b:8080/3,x", "http://a:8080/3,x"} - unreachable.Store("a:8080", time.Now().Add(-unreachableRetryInterval)) +func expireUnreachable(t *testing.T, host string) { + t.Helper() + v, found := unreachable.Load(host) + if !found { + t.Fatalf("%s is not recorded as unreachable", host) + } + failure := v.(unanswered) + failure.at = failure.at.Add(-unreachableRetryInterval) + unreachable.Store(host, failure) +} - // the first read to come by probes a, the next keeps it last until that settles +// waitProbe waits for the background probe started by ReachableFirst. +func waitProbe(host string) (stillUnreachable bool) { + probes.Wait() + _, stillUnreachable = unreachable.Load(host) + return stillUnreachable +} + +func TestReachableFirstProbesInBackgroundAfterRetryInterval(t *testing.T) { + forgetUnreachable(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer srv.Close() + host := srv.Listener.Addr().String() + urls := []string{"http://" + host + "/3,x", "http://b:8080/3,x"} + recordUnreachable(host, io.EOF) + expireUnreachable(t, host) + + // the read that finds the interval passed still tries the host last assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) - assertOrder(t, ReachableFirst(urls), urls...) - recordReachable("a:8080") + if waitProbe(host) { + t.Fatalf("an answering server stayed unreachable") + } assertOrder(t, ReachableFirst(urls), urls...) } -func TestReachableFirstProbesOneExpiredHostPerRead(t *testing.T) { +func TestReachableFirstKeepsHostLastWhenProbeFails(t *testing.T) { forgetUnreachable(t) - urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"} - expired := time.Now().Add(-unreachableRetryInterval) - unreachable.Store("a:8080", expired) - unreachable.Store("c:8080", expired) + refusing, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + refusing.Close() + silent, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer silent.Close() + go func() { + for { + conn, err := silent.Accept() + if err != nil { + return + } + defer conn.Close() + } + }() - assertOrder(t, ReachableFirst(urls), urls[0], urls[1], urls[2]) - assertOrder(t, ReachableFirst(urls), urls[2], urls[1], urls[0]) - assertOrder(t, ReachableFirst(urls), urls[1], urls[0], urls[2]) + for _, host := range []string{refusing.Addr().String(), silent.Addr().String()} { + urls := []string{"http://" + host + "/3,x", "http://b:8080/3,x"} + recordUnreachable(host, io.EOF) + expireUnreachable(t, host) + + assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) + if !waitProbe(host) { + t.Fatalf("%s was cleared by a probe it did not answer", host) + } + assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) + } +} + +func dialError(err error) error { + return &url.Error{Op: "Get", URL: "http://10.0.0.9:9501/3,x", Err: &net.OpError{Op: "dial", Net: "tcp", Err: err}} +} + +func TestReachableFirstTriesEveryServerOfAnUnansweringMachineLast(t *testing.T) { + for _, tc := range []struct{ host, sibling, other string }{ + {"10.0.0.9:9501", "10.0.0.9:9502", "10.0.0.8:9501"}, + {"10.0.0.9", "10.0.0.9:9502", "10.0.0.8:9501"}, + {"[2001:db8::9]:9501", "[2001:db8::9]", "[2001:db8::8]:9501"}, + {"[2001:db8::9]", "[2001:db8::9]:9502", "[2001:db8::8]:9501"}, + } { + t.Run(tc.host, func(t *testing.T) { + forgetUnreachable(t) + urls := []string{"http://" + tc.host + "/3,x", "http://" + tc.sibling + "/3,x", "http://" + tc.other + "/3,x"} + + recordUnreachable(tc.host, dialError(os.ErrDeadlineExceeded)) + assertOrder(t, ReachableFirst(urls), urls[2], urls[0], urls[1]) + + // a sibling port answering clears the machine, not the host that timed out + recordReachable(tc.sibling) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) + recordReachable(tc.host) + assertOrder(t, ReachableFirst(urls), urls...) + }) + } +} + +func refusedDial(t *testing.T) error { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := listener.Addr().String() + listener.Close() + conn, err := net.Dial("tcp", addr) + if err == nil { + conn.Close() + t.Skip("closed port accepted a connection") + } + if !connRefused(err) { + t.Fatalf("dial to a closed port: %v is not a refusal", err) + } + return &url.Error{Op: "Get", URL: "http://" + addr + "/3,x", Err: err} +} + +func TestReachableFirstKeepsOtherPortsFirstWhenOneRefuses(t *testing.T) { + forgetUnreachable(t) + urls := []string{"http://10.0.0.9:9501/3,x", "http://10.0.0.9:9502/3,x", "http://10.0.0.8:9501/3,x"} + refused := refusedDial(t) + + recordUnreachable("10.0.0.9:9501", refused) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) + + recordUnreachable("10.0.0.9:9502", dialError(os.ErrDeadlineExceeded)) + assertOrder(t, ReachableFirst(urls), urls[2], urls[0], urls[1]) + + // a refusal shows the machine answers again + forgetUnreachable(t) + recordUnreachable("10.0.0.9:9503", dialError(os.ErrDeadlineExceeded)) + recordUnreachable("10.0.0.9:9501", refused) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) } // hangupServer accepts the connection and drops it without answering, the way