diff --git a/weed/command/server.go b/weed/command/server.go index d5ec4a34b7..a7c8c99ccb 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -565,7 +565,8 @@ func idleTimeoutSeconds(seconds *int) time.Duration { func newHttpServer(h http.Handler, tlsConfig *tls.Config) *http.Server { s := &http.Server{ - Handler: h, + Handler: h, + ConnContext: util.ConnContext, } if tlsConfig != nil { s.TLSConfig = tlsConfig.Clone() diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index ff62601268..df56053810 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -293,7 +293,8 @@ func isCanceledStreamingError(err error) bool { // wrote a response. A canceled error is answered too: the idle timeout and a // shared lookup canceled by another request both cancel it while the client is // still connected, and returning silently would send that client an empty 200. -// A client that has gone away never sees the write. +// A client that has gone away never sees the write; it is answered +// ClientDisconnected so that an aborted download does not count as a 5xx. func writeStreamErrorResponse(w http.ResponseWriter, r *http.Request, err error) { var streamErr *StreamError if errors.As(err, &streamErr) && streamErr.ResponseWritten { @@ -303,6 +304,10 @@ func writeStreamErrorResponse(w http.ResponseWriter, r *http.Request, err error) s3err.WriteErrorResponse(w, r, s3err.ErrRequestBytesExceed) return } + if isCanceledStreamingError(err) && util.ClientGone(r.Context()) { + s3err.WriteErrorResponse(w, r, s3err.ErrClientDisconnected) + return + } s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) } diff --git a/weed/s3api/s3api_stream_error_test.go b/weed/s3api/s3api_stream_error_test.go index 2f8da45f84..8630451642 100644 --- a/weed/s3api/s3api_stream_error_test.go +++ b/weed/s3api/s3api_stream_error_test.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/chunk_cache" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" @@ -54,7 +55,7 @@ func TestIdleTimeoutCancelStillAnswersConnectedClient(t *testing.T) { if err != nil { t.Fatal(err) } - server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := &http.Server{ConnContext: util.ConnContext, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // a first read stalled past the idle timeout <-r.Context().Done() // only the read side should be short; a write timing out would fail the @@ -84,6 +85,79 @@ func TestIdleTimeoutCancelStillAnswersConnectedClient(t *testing.T) { } } +// TestClientGoneBeforeFirstByteIsNotAnInternalError checks that a canceled +// stream whose client went away is answered ClientDisconnected, so an aborted +// download does not count as a server error. +func TestClientGoneBeforeFirstByteIsNotAnInternalError(t *testing.T) { + clientCancels := func(cancel context.CancelFunc, _ *httptest.Server) { cancel() } + serverCloses := func(_ context.CancelFunc, server *httptest.Server) { server.CloseClientConnections() } + clientDisconnected := s3err.GetAPIError(s3err.ErrClientDisconnected).HTTPStatusCode + tests := []struct { + name string + start func(*httptest.Server) + abort func(context.CancelFunc, *httptest.Server) + want int + }{ + {"client closes the connection", (*httptest.Server).Start, clientCancels, clientDisconnected}, + {"client closes the TLS connection", (*httptest.Server).StartTLS, clientCancels, clientDisconnected}, + {"client resets the HTTP/2 stream", func(server *httptest.Server) { + server.EnableHTTP2 = true + server.StartTLS() + }, clientCancels, clientDisconnected}, + {"server closes the connection", (*httptest.Server).Start, serverCloses, http.StatusInternalServerError}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + listener, err := util.NewListener("127.0.0.1:0", time.Minute) + if err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + status := make(chan int, 1) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(entered) + <-r.Context().Done() + rec := httptest.NewRecorder() + writeStreamErrorResponse(rec, r, r.Context().Err()) + status <- rec.Code + })) + server.Listener.Close() + server.Listener = listener + server.Config.ConnContext = util.ConnContext + tt.start(server) + defer server.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/bucket/object", nil) + if err != nil { + t.Fatal(err) + } + go func() { + if resp, err := server.Client().Do(req); err == nil { + resp.Body.Close() + } + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("the request never reached the handler") + } + tt.abort(cancel, server) + + select { + case got := <-status: + if got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + case <-time.After(5 * time.Second): + t.Fatal("the request context was never canceled") + } + }) + } +} + // cancelingLookupFiler cancels the request context while the first chunk // lookup is in flight, as the idle timeout does to a client still connected. type cancelingLookupFiler struct { diff --git a/weed/util/net_timeout.go b/weed/util/net_timeout.go index eb26542b2c..e02e1a7cdd 100644 --- a/weed/util/net_timeout.go +++ b/weed/util/net_timeout.go @@ -1,8 +1,12 @@ package util import ( + "context" + "errors" "net" + "os" "sync" + "sync/atomic" "time" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -52,6 +56,8 @@ type Conn struct { mu sync.Mutex externalReadDeadline bool externalWriteDeadline bool + + readEndedByServer atomic.Bool } func (c *Conn) SetDeadline(t time.Time) error { @@ -107,13 +113,12 @@ func (c *Conn) extendDeadline() error { func (c *Conn) Read(b []byte) (count int, e error) { // Extend deadline before reading - any activity keeps connection alive - if err := c.extendDeadline(); err != nil { - return 0, err - } - count, e = c.Conn.Read(b) - if e == nil { - stats.BytesIn(int64(count)) + if e = c.extendDeadline(); e == nil { + if count, e = c.Conn.Read(b); e == nil { + stats.BytesIn(int64(count)) + } } + c.readEndedByServer.Store(errors.Is(e, os.ErrDeadlineExceeded) || errors.Is(e, net.ErrClosed)) return } @@ -140,6 +145,29 @@ func (c *Conn) Close() error { return err } +type connContextKey struct{} + +// ConnContext is an http.Server.ConnContext hook that lets ClientGone see the +// connection serving a request. +func ConnContext(ctx context.Context, c net.Conn) context.Context { + if tlsConn, ok := c.(interface{ NetConn() net.Conn }); ok { + c = tlsConn.NetConn() + } + if tc, ok := c.(*Conn); ok { + return context.WithValue(ctx, connContextKey{}, tc) + } + return ctx +} + +// ClientGone reports whether the client is why the request context ctx is +// canceled. net/http cancels it when a read of the connection fails or the +// HTTP/2 stream is reset; a read the server ended itself, by the idle timeout +// or by closing the connection, is not the client's doing. +func ClientGone(ctx context.Context) bool { + tc, _ := ctx.Value(connContextKey{}).(*Conn) + return tc != nil && ctx.Err() == context.Canceled && !tc.readEndedByServer.Load() +} + func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error) { listener, err := net.Listen("tcp", addr) if err != nil {