diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.go index c21e3df59e..9a4848811d 100644 --- a/weed/util/http/http_global_client_util.go +++ b/weed/util/http/http_global_client_util.go @@ -112,7 +112,7 @@ func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error response, err := GetGlobalHttpClient().Do(request) if err != nil { if ctx.Err() == nil { - recordUnreachable(request.URL.Host) + recordUnreachable(request.URL.Host, err) } return nil, true, err } @@ -401,7 +401,7 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, r, err := GetGlobalHttpClient().Do(req) if err != nil { if ctx.Err() == nil { - recordUnreachable(req.URL.Host) + recordUnreachable(req.URL.Host, err) } return true, err } @@ -801,7 +801,7 @@ func readUrlDirectToBuffer(ctx context.Context, fileUrl, jwt string, buffer []by r, err := GetGlobalHttpClient().Do(req) if err != nil { if ctx.Err() == nil { - recordUnreachable(req.URL.Host) + recordUnreachable(req.URL.Host, err) } return 0, true, err } diff --git a/weed/util/http/reachability.go b/weed/util/http/reachability.go index ad277b3eff..d97200183f 100644 --- a/weed/util/http/reachability.go +++ b/weed/util/http/reachability.go @@ -1,54 +1,119 @@ package http import ( + "context" + "errors" + "net" + "net/http" "net/url" + "strings" "sync" "time" ) // unreachableRetryInterval is how long a server that failed to answer is tried -// last before a read probes it again. +// last before it is probed again. const unreachableRetryInterval = 30 * time.Second -// unreachable maps a host to when it last failed to answer a request. A dead -// replica sits in the location list of every volume it held, so this is kept -// per host rather than per volume. -var unreachable sync.Map +// reachabilityProbeTimeout bounds the background request that checks whether +// an unanswering server is back. +const reachabilityProbeTimeout = 5 * time.Second -func recordUnreachable(host string) { - unreachable.Store(host, time.Now()) +type unanswered struct { + at time.Time + addr string } +// unreachable maps a host, and a machine whose dial got no answer, to its last +// failure. A dead replica sits in the location list of every volume it held, +// so this is kept per host rather than per volume. +var unreachable sync.Map + +// machineKey keys a machine apart from a host without an explicit port. +type machineKey string + +var probes sync.WaitGroup + +// recordUnreachable keys a failure by machine when the dial never reached it, +// so every server on a powered-off node is tried last after one failure. +func recordUnreachable(host string, err error) { + failure := unanswered{at: time.Now(), addr: host} + machine := machineOf(host) + unreachable.Store(host, failure) + if hostDown(err) { + unreachable.Store(machine, failure) + } else if connRefused(err) { + unreachable.Delete(machine) + } +} + +// recordReachable clears the machine verdict too, but a sibling port that +// failed on its own stays last until it answers. func recordReachable(host string) { unreachable.Delete(host) + unreachable.Delete(machineOf(host)) +} + +// hostDown reports whether a dial got no answer from the machine at all, unlike +// a refused connection, which shows the machine is up. +func hostDown(err error) bool { + var opErr *net.OpError + if !errors.As(err, &opErr) || opErr.Op != "dial" { + return false + } + return opErr.Timeout() || noRoute(err) } // ReachableFirst orders urls so that hosts which recently failed to answer // come last, keeping the order within each group. Once the retry interval has -// passed, the first read to ask claims the probe and tries that host first; -// the others keep it last until the probe settles. A read probes at most one -// host, so a claim is always followed by an attempt and several expired hosts -// are probed by successive reads. +// passed, such a host is probed in the background rather than by a read, so a +// dead replica never costs a read the dial timeout again. func ReachableFirst(urls []string) []string { - var probe, reachable, unanswered []string + var reachable, failed []string for _, u := range urls { - host := hostOf(u) - failedAt, failed := unreachable.Load(host) - switch { - case !failed: + if isUnanswering(hostOf(u)) { + failed = append(failed, u) + } else { reachable = append(reachable, u) - case time.Since(failedAt.(time.Time)) < unreachableRetryInterval: - unanswered = append(unanswered, u) - case probe == nil && unreachable.CompareAndSwap(host, failedAt, time.Now()): - probe = append(probe, u) - default: - unanswered = append(unanswered, u) } } - if probe == nil && unanswered == nil { + if failed == nil { return urls } - return append(append(probe, reachable...), unanswered...) + return append(reachable, failed...) +} + +func isUnanswering(host string) bool { + for _, key := range [2]any{host, machineOf(host)} { + v, found := unreachable.Load(key) + if !found { + continue + } + failure := v.(unanswered) + if time.Since(failure.at) >= unreachableRetryInterval && + unreachable.CompareAndSwap(key, failure, unanswered{at: time.Now(), addr: failure.addr}) { + probes.Go(func() { probeReachability(failure.addr) }) + } + return true + } + return false +} + +// probeReachability asks for response headers, not just a connection, so a +// server that accepts but never answers stays last. +func probeReachability(addr string) { + ctx, cancel := context.WithTimeout(context.Background(), reachabilityProbeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+addr+"/status", nil) + if err == nil { + var resp *http.Response + if resp, err = GetGlobalHttpClient().Do(req); err == nil { + resp.Body.Close() + recordReachable(addr) + return + } + } + recordUnreachable(addr, err) } func hostOf(rawUrl string) string { @@ -58,3 +123,13 @@ func hostOf(rawUrl string) string { } return parsed.Host } + +func machineOf(host string) machineKey { + if machine, _, err := net.SplitHostPort(host); err == nil { + return machineKey(machine) + } + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = host[1 : len(host)-1] + } + return machineKey(host) +} diff --git a/weed/util/http/reachability_errno.go b/weed/util/http/reachability_errno.go new file mode 100644 index 0000000000..8856557807 --- /dev/null +++ b/weed/util/http/reachability_errno.go @@ -0,0 +1,16 @@ +//go:build !windows + +package http + +import ( + "errors" + "syscall" +) + +func connRefused(err error) bool { + return errors.Is(err, syscall.ECONNREFUSED) +} + +func noRoute(err error) bool { + return errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTDOWN) +} diff --git a/weed/util/http/reachability_errno_windows.go b/weed/util/http/reachability_errno_windows.go new file mode 100644 index 0000000000..18cee5b260 --- /dev/null +++ b/weed/util/http/reachability_errno_windows.go @@ -0,0 +1,15 @@ +package http + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +func connRefused(err error) bool { + return errors.Is(err, windows.WSAECONNREFUSED) +} + +func noRoute(err error) bool { + return errors.Is(err, windows.WSAEHOSTUNREACH) || errors.Is(err, windows.WSAENETUNREACH) || errors.Is(err, windows.WSAEHOSTDOWN) +} diff --git a/weed/util/http/reachability_test.go b/weed/util/http/reachability_test.go index 4901ede9bb..ad7c5149d9 100644 --- a/weed/util/http/reachability_test.go +++ b/weed/util/http/reachability_test.go @@ -2,11 +2,14 @@ package http import ( "context" + "io" + "net" "net/http" "net/http/httptest" + "net/url" + "os" "sync/atomic" "testing" - "time" ) func forgetUnreachable(t *testing.T) { @@ -18,7 +21,10 @@ func forgetUnreachable(t *testing.T) { }) } forget() - t.Cleanup(forget) + t.Cleanup(func() { + probes.Wait() + forget() + }) } func assertOrder(t *testing.T, got []string, want ...string) { @@ -38,34 +44,146 @@ func TestReachableFirstTriesUnansweringHostLast(t *testing.T) { urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"} assertOrder(t, ReachableFirst(urls), urls...) - recordUnreachable("a:8080") + recordUnreachable("a:8080", io.EOF) assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) recordReachable("a:8080") assertOrder(t, ReachableFirst(urls), urls...) } -func TestReachableFirstProbesOnceAfterRetryInterval(t *testing.T) { - forgetUnreachable(t) - urls := []string{"http://b:8080/3,x", "http://a:8080/3,x"} - unreachable.Store("a:8080", time.Now().Add(-unreachableRetryInterval)) +func expireUnreachable(t *testing.T, host string) { + t.Helper() + v, found := unreachable.Load(host) + if !found { + t.Fatalf("%s is not recorded as unreachable", host) + } + failure := v.(unanswered) + failure.at = failure.at.Add(-unreachableRetryInterval) + unreachable.Store(host, failure) +} - // the first read to come by probes a, the next keeps it last until that settles +// waitProbe waits for the background probe started by ReachableFirst. +func waitProbe(host string) (stillUnreachable bool) { + probes.Wait() + _, stillUnreachable = unreachable.Load(host) + return stillUnreachable +} + +func TestReachableFirstProbesInBackgroundAfterRetryInterval(t *testing.T) { + forgetUnreachable(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer srv.Close() + host := srv.Listener.Addr().String() + urls := []string{"http://" + host + "/3,x", "http://b:8080/3,x"} + recordUnreachable(host, io.EOF) + expireUnreachable(t, host) + + // the read that finds the interval passed still tries the host last assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) - assertOrder(t, ReachableFirst(urls), urls...) - recordReachable("a:8080") + if waitProbe(host) { + t.Fatalf("an answering server stayed unreachable") + } assertOrder(t, ReachableFirst(urls), urls...) } -func TestReachableFirstProbesOneExpiredHostPerRead(t *testing.T) { +func TestReachableFirstKeepsHostLastWhenProbeFails(t *testing.T) { forgetUnreachable(t) - urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"} - expired := time.Now().Add(-unreachableRetryInterval) - unreachable.Store("a:8080", expired) - unreachable.Store("c:8080", expired) + refusing, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + refusing.Close() + silent, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer silent.Close() + go func() { + for { + conn, err := silent.Accept() + if err != nil { + return + } + defer conn.Close() + } + }() - assertOrder(t, ReachableFirst(urls), urls[0], urls[1], urls[2]) - assertOrder(t, ReachableFirst(urls), urls[2], urls[1], urls[0]) - assertOrder(t, ReachableFirst(urls), urls[1], urls[0], urls[2]) + for _, host := range []string{refusing.Addr().String(), silent.Addr().String()} { + urls := []string{"http://" + host + "/3,x", "http://b:8080/3,x"} + recordUnreachable(host, io.EOF) + expireUnreachable(t, host) + + assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) + if !waitProbe(host) { + t.Fatalf("%s was cleared by a probe it did not answer", host) + } + assertOrder(t, ReachableFirst(urls), urls[1], urls[0]) + } +} + +func dialError(err error) error { + return &url.Error{Op: "Get", URL: "http://10.0.0.9:9501/3,x", Err: &net.OpError{Op: "dial", Net: "tcp", Err: err}} +} + +func TestReachableFirstTriesEveryServerOfAnUnansweringMachineLast(t *testing.T) { + for _, tc := range []struct{ host, sibling, other string }{ + {"10.0.0.9:9501", "10.0.0.9:9502", "10.0.0.8:9501"}, + {"10.0.0.9", "10.0.0.9:9502", "10.0.0.8:9501"}, + {"[2001:db8::9]:9501", "[2001:db8::9]", "[2001:db8::8]:9501"}, + {"[2001:db8::9]", "[2001:db8::9]:9502", "[2001:db8::8]:9501"}, + } { + t.Run(tc.host, func(t *testing.T) { + forgetUnreachable(t) + urls := []string{"http://" + tc.host + "/3,x", "http://" + tc.sibling + "/3,x", "http://" + tc.other + "/3,x"} + + recordUnreachable(tc.host, dialError(os.ErrDeadlineExceeded)) + assertOrder(t, ReachableFirst(urls), urls[2], urls[0], urls[1]) + + // a sibling port answering clears the machine, not the host that timed out + recordReachable(tc.sibling) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) + recordReachable(tc.host) + assertOrder(t, ReachableFirst(urls), urls...) + }) + } +} + +func refusedDial(t *testing.T) error { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := listener.Addr().String() + listener.Close() + conn, err := net.Dial("tcp", addr) + if err == nil { + conn.Close() + t.Skip("closed port accepted a connection") + } + if !connRefused(err) { + t.Fatalf("dial to a closed port: %v is not a refusal", err) + } + return &url.Error{Op: "Get", URL: "http://" + addr + "/3,x", Err: err} +} + +func TestReachableFirstKeepsOtherPortsFirstWhenOneRefuses(t *testing.T) { + forgetUnreachable(t) + urls := []string{"http://10.0.0.9:9501/3,x", "http://10.0.0.9:9502/3,x", "http://10.0.0.8:9501/3,x"} + refused := refusedDial(t) + + recordUnreachable("10.0.0.9:9501", refused) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) + + recordUnreachable("10.0.0.9:9502", dialError(os.ErrDeadlineExceeded)) + assertOrder(t, ReachableFirst(urls), urls[2], urls[0], urls[1]) + + // a refusal shows the machine answers again + forgetUnreachable(t) + recordUnreachable("10.0.0.9:9503", dialError(os.ErrDeadlineExceeded)) + recordUnreachable("10.0.0.9:9501", refused) + assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0]) } // hangupServer accepts the connection and drops it without answering, the way