mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
938a15eb98
ObjectTransaction forwards to the ring owner so one filer's per-path lock arbitrates every writer of a key. But a ring change hands the key over before the new owner has rebuilt the locks the prior owner still holds, so for the cooling-off window both can grant it. LockRing.PriorOwner exists for exactly this and nothing consulted it. Route to the prior owner while that window is open. LockRing.WriteOwner resolves prior-else-current under one read lock, so the pair cannot come from different rings and name the same filer twice. An unreachable owner fails the request rather than falling back to the current one. gRPC reports a response lost in transit as Unavailable, indistinguishable from a request the owner never saw, so re-sending elsewhere could re-apply what the owner already committed; and an owner unreachable from here may be partitioned rather than down, still serving the key to everyone else — which is the split brain the routing exists to prevent. The window is bounded: once it closes the ring hands the key to its new owner. The owner resolution and the forward move into writeOwner/forwardToWriteOwner so the next routed RPC reuses them rather than copying the block. Claude-Session: https://claude.ai/code/session_01Fx1Hx8RqsJqHpbfbgTf4WJ
109 lines
3.7 KiB
Go
109 lines
3.7 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
)
|
|
|
|
// routeTestServer builds a filer whose ring holds the given snapshots, newest
|
|
// last. Host is deliberately outside the ring in the forwarding tests so no key
|
|
// is ever owned locally.
|
|
func routeTestServer(host pb.ServerAddress, snapshots ...[]pb.ServerAddress) *FilerServer {
|
|
dlm := lock_manager.NewDistributedLockManager(host)
|
|
for i, servers := range snapshots {
|
|
dlm.LockRing.SetSnapshot(servers, int64(i+1))
|
|
}
|
|
return &FilerServer{
|
|
filer: &filer.Filer{Dlm: dlm},
|
|
option: &FilerOption{Host: host},
|
|
}
|
|
}
|
|
|
|
var (
|
|
routeSetA = []pb.ServerAddress{"f1:8888", "f2:8888", "f3:8888"}
|
|
routeSetB = []pb.ServerAddress{"f1:8888", "f2:8888", "f3:8888", "f4:8888"}
|
|
)
|
|
|
|
// firstKey returns the first synthetic path the ring answers match for.
|
|
func firstKey(t *testing.T, match func(key string) bool) string {
|
|
t.Helper()
|
|
for i := 0; i < 4000; i++ {
|
|
key := fmt.Sprintf("/buckets/b/key-%d", i)
|
|
if match(key) {
|
|
return key
|
|
}
|
|
}
|
|
t.Skip("no key satisfying the ring condition")
|
|
return ""
|
|
}
|
|
|
|
func TestWriteOwnerNoRing(t *testing.T) {
|
|
fs := routeTestServer("f9:8888")
|
|
if got := fs.writeOwner("/any"); got != "" {
|
|
t.Fatalf("no ring snapshot must leave the write local, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestWriteOwnerStableKeyUsesPrimary(t *testing.T) {
|
|
fs := routeTestServer("f9:8888", routeSetA, routeSetB)
|
|
ring := fs.filer.Dlm.LockRing
|
|
|
|
stable := firstKey(t, func(key string) bool { return ring.PriorOwner(key) == "" })
|
|
if got := fs.writeOwner(stable); got != ring.GetPrimary(stable) {
|
|
t.Fatalf("stable key must route to the primary, got %v want %v", got, ring.GetPrimary(stable))
|
|
}
|
|
}
|
|
|
|
// A key whose ownership just moved must keep going to the prior owner: the new
|
|
// owner has not rebuilt the locks the prior one still holds.
|
|
func TestWriteOwnerMovedKeyUsesPriorOwner(t *testing.T) {
|
|
fs := routeTestServer("f9:8888", routeSetA, routeSetB)
|
|
ring := fs.filer.Dlm.LockRing
|
|
|
|
moved := firstKey(t, func(key string) bool { return ring.PriorOwner(key) != "" })
|
|
if got := fs.writeOwner(moved); got != ring.PriorOwner(moved) {
|
|
t.Fatalf("moved key must route to the prior owner, got %v want %v", got, ring.PriorOwner(moved))
|
|
}
|
|
}
|
|
|
|
func TestForwardToWriteOwnerAppliesLocallyWhenOwned(t *testing.T) {
|
|
fs := routeTestServer("f2:8888", routeSetA)
|
|
ring := fs.filer.Dlm.LockRing
|
|
|
|
key := firstKey(t, func(key string) bool { return ring.GetPrimary(key) == "f2:8888" })
|
|
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(pb.ServerAddress) error {
|
|
t.Fatal("must not forward a key this filer owns")
|
|
return nil
|
|
})
|
|
if handled || err != nil {
|
|
t.Fatalf("owned key must be applied locally, got handled=%v err=%v", handled, err)
|
|
}
|
|
}
|
|
|
|
// A failed forward must surface, never re-send to a second filer: gRPC cannot
|
|
// tell a lost response from an unsent request, and an owner unreachable from
|
|
// here may be partitioned rather than down.
|
|
func TestForwardToWriteOwnerNeverTriesASecondFiler(t *testing.T) {
|
|
fs := routeTestServer("f9:8888", routeSetA, routeSetB)
|
|
ring := fs.filer.Dlm.LockRing
|
|
|
|
moved := firstKey(t, func(key string) bool { return ring.PriorOwner(key) != "" })
|
|
var tried []pb.ServerAddress
|
|
handled, err := fs.forwardToWriteOwner(context.Background(), moved, func(owner pb.ServerAddress) error {
|
|
tried = append(tried, owner)
|
|
return errors.New("dial tcp: connection refused")
|
|
})
|
|
if !handled || err == nil {
|
|
t.Fatalf("unreachable owner must surface an error, got handled=%v err=%v", handled, err)
|
|
}
|
|
if len(tried) != 1 || tried[0] != ring.PriorOwner(moved) {
|
|
t.Fatalf("expected exactly the prior owner, tried %v", tried)
|
|
}
|
|
}
|