mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-11 17:10:40 +02:00
Chris Lu
f037fc4dce
* s3: dial the object lock's primary filer directly The S3 object write lock builds a fresh short-lived lock per write, each starting at the seed filer. When the seed isn't the key's hash-ring primary the filer forwards the request to the primary, and in multi-cluster setups that forward crosses clusters on every write. Give the lock client a view of the filer lock ring, fed by the master's LockRingUpdate broadcasts the gateway already receives, so it dials the primary directly. The view tracks filer membership by version; a stale view stays correct because the filer still forwards as a fallback. Also send the initial ring snapshot to S3 clients, not just filers. * s3: subscribe to lock-ring updates before starting the master loop The master delivers the initial LockRingUpdate once, on connect. Registering the callback after KeepConnectedToMaster started left a window where that first update could arrive before the handler was set and be dropped, delaying the ring view until the next membership change. Build the lock client and register the callback in the masters block before launching the loop; the filers block reuses that client (or creates a plain one when no masters are configured). * lock_manager: build the hash ring in a deterministic server order rebuildRing ranged over the server set (a map), whose iteration order is randomized per process. On a vnode hash collision the last writer into vnodeToServer wins, so two nodes holding the same server set could resolve the collision to different servers and disagree on the primary for keys near that slot. Now that the S3 gateway also computes PrimaryForKey, such a disagreement would route the same key to different filers and defeat per-path serialization. Iterate the servers in sorted order so the ring is identical on every node with the same set, regardless of discovery order. * lock_manager: skip redundant ring rebuilds, trim comments SetRing now ignores a non-zero version at or below the current one once a ring exists, so repeated LockRingUpdate broadcasts on reconnect no longer rebuild the ring. * s3: hold the lock-ring client on the server for route-by-key Store the object-write lock client on S3ApiServer so handlers can resolve a key's owner filer via PrimaryForKey.
93 lines
3.1 KiB
Go
93 lines
3.1 KiB
Go
package cluster
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
)
|
|
|
|
// The gateway must resolve a lock key to the same primary the filers do,
|
|
// otherwise it dials the wrong filer and the lock still gets forwarded. Both
|
|
// sides use the same HashRing over the same server set, so for every key the
|
|
// client's hostForKey must equal the filer ring's GetPrimary.
|
|
func TestLockClientHostMatchesFilerRing(t *testing.T) {
|
|
servers := []pb.ServerAddress{
|
|
"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888",
|
|
}
|
|
|
|
filerRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
|
|
filerRing.SetServers(servers)
|
|
|
|
lc := NewLockClient(nil, "seed:8888")
|
|
lc.SetRing(servers, 1)
|
|
|
|
for _, key := range []string{
|
|
"s3.object.write:/buckets/b/obj-0",
|
|
"s3.object.write:/buckets/b/obj-1",
|
|
"s3.object.write:/buckets/b/obj-2",
|
|
"s3.object.write:/buckets/gosbench-0/w0obj-kilo-0877",
|
|
"some/other/key",
|
|
} {
|
|
if got, want := lc.hostForKey(key), filerRing.GetPrimary(key); got != want {
|
|
t.Errorf("key %q: client host %q != filer primary %q", key, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Without a ring view, the client falls back to the seed filer (which the filer
|
|
// forwards from), preserving the pre-optimization behavior.
|
|
func TestLockClientHostFallsBackToSeed(t *testing.T) {
|
|
lc := NewLockClient(nil, "seed:8888")
|
|
if got := lc.hostForKey("any-key"); got != "seed:8888" {
|
|
t.Errorf("expected seed fallback, got %q", got)
|
|
}
|
|
|
|
// An empty ring (no members yet) also falls back to the seed.
|
|
lc.SetRing(nil, 1)
|
|
if got := lc.hostForKey("any-key"); got != "seed:8888" {
|
|
t.Errorf("expected seed fallback on empty ring, got %q", got)
|
|
}
|
|
}
|
|
|
|
// A stale (older-version) update must not regress a newer ring view, while
|
|
// version 0 always applies as a bootstrap.
|
|
func TestLockClientSetRingVersionGuard(t *testing.T) {
|
|
lc := NewLockClient(nil, "seed:8888")
|
|
|
|
newer := []pb.ServerAddress{"filer-a:8888", "filer-b:8888"}
|
|
lc.SetRing(newer, 10)
|
|
primaryAt10 := lc.hostForKey("k")
|
|
|
|
// Older version is ignored.
|
|
lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 5)
|
|
if got := lc.hostForKey("k"); got != primaryAt10 {
|
|
t.Errorf("stale update applied: host changed to %q", got)
|
|
}
|
|
|
|
// version 0 is always accepted.
|
|
lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 0)
|
|
if got := lc.hostForKey("k"); got != "filer-z:8888" {
|
|
t.Errorf("bootstrap update not applied, got %q", got)
|
|
}
|
|
}
|
|
|
|
// PrimaryForKey returns "" before any ring is received (so a route-by-key
|
|
// caller falls back to the distributed lock) and the ring owner afterwards,
|
|
// unlike hostForKey which falls back to the seed.
|
|
func TestLockClientPrimaryForKey(t *testing.T) {
|
|
lc := NewLockClient(nil, "seed:8888")
|
|
if got := lc.PrimaryForKey("k"); got != "" {
|
|
t.Errorf("expected empty before ring, got %q", got)
|
|
}
|
|
|
|
lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 1)
|
|
got := lc.PrimaryForKey("k")
|
|
if got == "" {
|
|
t.Fatal("expected an owner after ring set")
|
|
}
|
|
if got != lc.hostForKey("k") {
|
|
t.Errorf("PrimaryForKey %q disagrees with hostForKey %q", got, lc.hostForKey("k"))
|
|
}
|
|
}
|