mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-13 17:48:29 +02:00
Chris LuandDevin
6d692ab013
* mount: bound each filer call so a wedged filer cannot hold the failover walk WithFilerClient callers run their RPCs on context.Background(), so a filer whose transport stalls without refusing (stopped peer, black-holed connection) blocks the call forever and the walk never reaches the remaining filer addresses. Run each attempt under a timeout; on expiry drop the cached channel so the abandoned call is unblocked and the next address is tried. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * mount: time out wedged mutation streams and rotate to the next filer doUnary and Rename wait on stream Send/Recv under context.Background(); a dead transport never delivers an error, so writes hang without ever reaching the unary fallback. Bound the waits and tear the stream down on timeout; teardown also advances the sticky filer index so the next stream opens against a different filer instead of redialing the dead one. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * mount: cover filer failover past a wedged filer Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * mount: cancel filer attempts by context and unwind them before retry Review follow-ups on the filer failover: - callFiler derives a per-attempt context instead of only abandoning the wait, so a timed-out attempt's RPCs die on their own without tearing down the shared cached channel. The walk now also waits for the attempt's callback to unwind before trying the next filer, so two runs of the same closure cannot race on the caller's captured results. Callbacks that ignore the attempt context are still forced out by dropping the channel, as a last resort after a grace period. - teardownStream captures the dying stream's filer index and rotates it while holding m.mu, so it can never rotate a successor generation's filer by mistake. - The unary rename fallback bounds silence on its dedicated stream the way the mux does, instead of trusting Recv on a wedged filer. - downloadRemoteEntry opts out of the metadata-call bound: a healthy whole-file download can legitimately outlast it. * mount: count only wire silence against the rename timeout Applying a rename event can outlast the silence bound on a healthy filer — a directory rename migrates an open child's DLM lock under another writer. Stop the mux's response timer for the local work and re-arm it before the next wait; the fallback stream's watchdog defers while the handler runs instead of cancelling a live stream. * mount: restart the rename fallback's silence bound after each event The bound covers the wait for the next response, not the rename's whole lifetime — a healthy rename that keeps sending events can outlast it. Re-arm after each handled event; the defer-while-handling stays for local work that outlasts the interval on its own. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
78 lines
3.0 KiB
Go
78 lines
3.0 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"syscall"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
// streamCreateEntry routes a CreateEntryRequest through the streaming mux
|
|
// if available, falling back to a unary gRPC call on transport errors.
|
|
func (wfs *WFS) streamCreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
|
|
if wfs.streamMutate != nil && wfs.streamMutate.IsAvailable() {
|
|
resp, err := wfs.streamMutate.CreateEntry(ctx, req)
|
|
if err == nil || !errors.Is(err, ErrStreamTransport) {
|
|
return resp, err // success or application error — don't retry
|
|
}
|
|
glog.V(1).Infof("streamCreateEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Entry.Name, err)
|
|
}
|
|
var resp *filer_pb.CreateEntryResponse
|
|
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
|
var err error
|
|
resp, err = filer_pb.CreateEntryWithResponse(ctx, client, req)
|
|
return err
|
|
})
|
|
return resp, err
|
|
}
|
|
|
|
// streamUpdateEntry routes an UpdateEntryRequest through the streaming mux
|
|
// if available, falling back to a unary gRPC call on transport errors.
|
|
func (wfs *WFS) streamUpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
|
|
if wfs.streamMutate != nil && wfs.streamMutate.IsAvailable() {
|
|
resp, err := wfs.streamMutate.UpdateEntry(ctx, req)
|
|
if err == nil || !errors.Is(err, ErrStreamTransport) {
|
|
return resp, err
|
|
}
|
|
glog.V(1).Infof("streamUpdateEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Entry.Name, err)
|
|
}
|
|
var resp *filer_pb.UpdateEntryResponse
|
|
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
|
var err error
|
|
resp, err = client.UpdateEntry(ctx, req)
|
|
return err
|
|
})
|
|
return resp, err
|
|
}
|
|
|
|
// streamDeleteEntry routes a DeleteEntryRequest through the streaming mux
|
|
// if available, falling back to a unary gRPC call on transport errors.
|
|
func (wfs *WFS) streamDeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
|
|
if wfs.streamMutate != nil && wfs.streamMutate.IsAvailable() {
|
|
resp, err := wfs.streamMutate.DeleteEntry(ctx, req)
|
|
if err == nil || !errors.Is(err, ErrStreamTransport) {
|
|
return resp, err
|
|
}
|
|
glog.V(1).Infof("streamDeleteEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Name, err)
|
|
}
|
|
var resp *filer_pb.DeleteEntryResponse
|
|
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
|
var err error
|
|
resp, err = client.DeleteEntry(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// A refused delete arrives in the response, as the stream branch above.
|
|
if resp.Error != "" {
|
|
return &streamMutateError{msg: resp.Error, errno: syscall.EIO}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return resp, nil
|
|
}
|