mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 17:55:26 +01:00
s3: answer ClientDisconnected for a GET the client aborted before the first byte (#12707)
* s3: answer ClientDisconnected for a GET the client aborted before the first byte Fixes #12706 * s3: tell a gone client on TLS and HTTP/2 too, and not on a connection the server closed
This commit is contained in:
1 parent
c91da3d1f8
commit
91eaabf371
4 files changed
+117
-9
No files matched your search
@@ -565,7 +565,8 @@ func idleTimeoutSeconds(seconds *int) time.Duration {
|
|||||||
|
|
||||||
func newHttpServer(h http.Handler, tlsConfig *tls.Config) *http.Server {
|
func newHttpServer(h http.Handler, tlsConfig *tls.Config) *http.Server {
|
||||||
s := &http.Server{
|
s := &http.Server{
|
||||||
Handler: h,
|
Handler: h,
|
||||||
|
ConnContext: util.ConnContext,
|
||||||
}
|
}
|
||||||
if tlsConfig != nil {
|
if tlsConfig != nil {
|
||||||
s.TLSConfig = tlsConfig.Clone()
|
s.TLSConfig = tlsConfig.Clone()
|
||||||
|
|||||||
@@ -293,7 +293,8 @@ func isCanceledStreamingError(err error) bool {
|
|||||||
// wrote a response. A canceled error is answered too: the idle timeout and a
|
// wrote a response. A canceled error is answered too: the idle timeout and a
|
||||||
// shared lookup canceled by another request both cancel it while the client is
|
// shared lookup canceled by another request both cancel it while the client is
|
||||||
// still connected, and returning silently would send that client an empty 200.
|
// still connected, and returning silently would send that client an empty 200.
|
||||||
// A client that has gone away never sees the write.
|
// A client that has gone away never sees the write; it is answered
|
||||||
|
// ClientDisconnected so that an aborted download does not count as a 5xx.
|
||||||
func writeStreamErrorResponse(w http.ResponseWriter, r *http.Request, err error) {
|
func writeStreamErrorResponse(w http.ResponseWriter, r *http.Request, err error) {
|
||||||
var streamErr *StreamError
|
var streamErr *StreamError
|
||||||
if errors.As(err, &streamErr) && streamErr.ResponseWritten {
|
if errors.As(err, &streamErr) && streamErr.ResponseWritten {
|
||||||
@@ -303,6 +304,10 @@ func writeStreamErrorResponse(w http.ResponseWriter, r *http.Request, err error)
|
|||||||
s3err.WriteErrorResponse(w, r, s3err.ErrRequestBytesExceed)
|
s3err.WriteErrorResponse(w, r, s3err.ErrRequestBytesExceed)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if isCanceledStreamingError(err) && util.ClientGone(r.Context()) {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrClientDisconnected)
|
||||||
|
return
|
||||||
|
}
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
|
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
|
||||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||||
@@ -54,7 +55,7 @@ func TestIdleTimeoutCancelStillAnswersConnectedClient(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := &http.Server{ConnContext: util.ConnContext, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
// a first read stalled past the idle timeout
|
// a first read stalled past the idle timeout
|
||||||
<-r.Context().Done()
|
<-r.Context().Done()
|
||||||
// only the read side should be short; a write timing out would fail the
|
// only the read side should be short; a write timing out would fail the
|
||||||
@@ -84,6 +85,79 @@ func TestIdleTimeoutCancelStillAnswersConnectedClient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestClientGoneBeforeFirstByteIsNotAnInternalError checks that a canceled
|
||||||
|
// stream whose client went away is answered ClientDisconnected, so an aborted
|
||||||
|
// download does not count as a server error.
|
||||||
|
func TestClientGoneBeforeFirstByteIsNotAnInternalError(t *testing.T) {
|
||||||
|
clientCancels := func(cancel context.CancelFunc, _ *httptest.Server) { cancel() }
|
||||||
|
serverCloses := func(_ context.CancelFunc, server *httptest.Server) { server.CloseClientConnections() }
|
||||||
|
clientDisconnected := s3err.GetAPIError(s3err.ErrClientDisconnected).HTTPStatusCode
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
start func(*httptest.Server)
|
||||||
|
abort func(context.CancelFunc, *httptest.Server)
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"client closes the connection", (*httptest.Server).Start, clientCancels, clientDisconnected},
|
||||||
|
{"client closes the TLS connection", (*httptest.Server).StartTLS, clientCancels, clientDisconnected},
|
||||||
|
{"client resets the HTTP/2 stream", func(server *httptest.Server) {
|
||||||
|
server.EnableHTTP2 = true
|
||||||
|
server.StartTLS()
|
||||||
|
}, clientCancels, clientDisconnected},
|
||||||
|
{"server closes the connection", (*httptest.Server).Start, serverCloses, http.StatusInternalServerError},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
listener, err := util.NewListener("127.0.0.1:0", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entered := make(chan struct{})
|
||||||
|
status := make(chan int, 1)
|
||||||
|
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
close(entered)
|
||||||
|
<-r.Context().Done()
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
writeStreamErrorResponse(rec, r, r.Context().Err())
|
||||||
|
status <- rec.Code
|
||||||
|
}))
|
||||||
|
server.Listener.Close()
|
||||||
|
server.Listener = listener
|
||||||
|
server.Config.ConnContext = util.ConnContext
|
||||||
|
tt.start(server)
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/bucket/object", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if resp, err := server.Client().Do(req); err == nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the request never reached the handler")
|
||||||
|
}
|
||||||
|
tt.abort(cancel, server)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-status:
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("status = %d, want %d", got, tt.want)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the request context was never canceled")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// cancelingLookupFiler cancels the request context while the first chunk
|
// cancelingLookupFiler cancels the request context while the first chunk
|
||||||
// lookup is in flight, as the idle timeout does to a client still connected.
|
// lookup is in flight, as the idle timeout does to a client still connected.
|
||||||
type cancelingLookupFiler struct {
|
type cancelingLookupFiler struct {
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
package util
|
package util
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
@@ -52,6 +56,8 @@ type Conn struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
externalReadDeadline bool
|
externalReadDeadline bool
|
||||||
externalWriteDeadline bool
|
externalWriteDeadline bool
|
||||||
|
|
||||||
|
readEndedByServer atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Conn) SetDeadline(t time.Time) error {
|
func (c *Conn) SetDeadline(t time.Time) error {
|
||||||
@@ -107,13 +113,12 @@ func (c *Conn) extendDeadline() error {
|
|||||||
|
|
||||||
func (c *Conn) Read(b []byte) (count int, e error) {
|
func (c *Conn) Read(b []byte) (count int, e error) {
|
||||||
// Extend deadline before reading - any activity keeps connection alive
|
// Extend deadline before reading - any activity keeps connection alive
|
||||||
if err := c.extendDeadline(); err != nil {
|
if e = c.extendDeadline(); e == nil {
|
||||||
return 0, err
|
if count, e = c.Conn.Read(b); e == nil {
|
||||||
}
|
stats.BytesIn(int64(count))
|
||||||
count, e = c.Conn.Read(b)
|
}
|
||||||
if e == nil {
|
|
||||||
stats.BytesIn(int64(count))
|
|
||||||
}
|
}
|
||||||
|
c.readEndedByServer.Store(errors.Is(e, os.ErrDeadlineExceeded) || errors.Is(e, net.ErrClosed))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,6 +145,29 @@ func (c *Conn) Close() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type connContextKey struct{}
|
||||||
|
|
||||||
|
// ConnContext is an http.Server.ConnContext hook that lets ClientGone see the
|
||||||
|
// connection serving a request.
|
||||||
|
func ConnContext(ctx context.Context, c net.Conn) context.Context {
|
||||||
|
if tlsConn, ok := c.(interface{ NetConn() net.Conn }); ok {
|
||||||
|
c = tlsConn.NetConn()
|
||||||
|
}
|
||||||
|
if tc, ok := c.(*Conn); ok {
|
||||||
|
return context.WithValue(ctx, connContextKey{}, tc)
|
||||||
|
}
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClientGone reports whether the client is why the request context ctx is
|
||||||
|
// canceled. net/http cancels it when a read of the connection fails or the
|
||||||
|
// HTTP/2 stream is reset; a read the server ended itself, by the idle timeout
|
||||||
|
// or by closing the connection, is not the client's doing.
|
||||||
|
func ClientGone(ctx context.Context) bool {
|
||||||
|
tc, _ := ctx.Value(connContextKey{}).(*Conn)
|
||||||
|
return tc != nil && ctx.Err() == context.Canceled && !tc.readEndedByServer.Load()
|
||||||
|
}
|
||||||
|
|
||||||
func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error) {
|
func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error) {
|
||||||
listener, err := net.Listen("tcp", addr)
|
listener, err := net.Listen("tcp", addr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Reference in new issue
Block a user