mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
wdclient: keep the embedded S3 gateway on its local filer across discovery (#12771)
* filer: keep an explicit gRPC port in the embedded gateways' filer address runFiler handed the embedded S3/WebDAV/IAM/SFTP gateways NewServerAddress(...).String(), and String() drops the .grpcPort part. With a non-default -port.grpc the gateways then dialed port+10000 and did not recognize their own filer in the master's list. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * wdclient: match discovered filers by gRPC address A filer listed as host:port and the same filer discovered as host:port.grpcPort were compared as raw strings, so a refresh pruned the configured filer, added every discovered filer as new in map order, and reset the current filer to whichever came first; a pushed join added it a second time. Key filers by their gRPC address: a survivor keeps its health and current position and takes the discovered address form, and pushed joins and leaves match the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c1c95e127f
commit
05a1463855
3 files changed
+134
-31
No files matched your search
@@ -307,7 +307,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
// -ip.bind is unset and the bind address falls back to -ip. This must be
|
||||
// captured before startFiler fills *f.bindIp in place.
|
||||
filerBindIpUnset := *f.bindIp == ""
|
||||
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
|
||||
filerAddress := string(pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc))
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
|
||||
@@ -230,7 +230,7 @@ func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) {
|
||||
|
||||
// Find the index of the specified filer address
|
||||
for i, filer := range fc.filerAddresses {
|
||||
if filer == addr {
|
||||
if filerKey(filer) == filerKey(addr) {
|
||||
atomic.StoreInt32(&fc.filerIndex, int32(i))
|
||||
return
|
||||
}
|
||||
@@ -246,7 +246,7 @@ func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool {
|
||||
|
||||
// Find the health for this filer address
|
||||
for i, filer := range fc.filerAddresses {
|
||||
if filer == addr {
|
||||
if filerKey(filer) == filerKey(addr) {
|
||||
if i < len(fc.filerHealth) {
|
||||
return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i])
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) {
|
||||
|
||||
// Find the health for this filer address
|
||||
for i, filer := range fc.filerAddresses {
|
||||
if filer == addr {
|
||||
if filerKey(filer) == filerKey(addr) {
|
||||
if i < len(fc.filerHealth) {
|
||||
fc.recordFilerSuccessWithHealth(fc.filerHealth[i])
|
||||
}
|
||||
@@ -280,7 +280,7 @@ func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) {
|
||||
|
||||
// Find the health for this filer address
|
||||
for i, filer := range fc.filerAddresses {
|
||||
if filer == addr {
|
||||
if filerKey(filer) == filerKey(addr) {
|
||||
if i < len(fc.filerHealth) {
|
||||
fc.recordFilerFailureWithHealth(fc.filerHealth[i])
|
||||
}
|
||||
@@ -413,12 +413,16 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time.
|
||||
fc.filerAddressesMu.Lock()
|
||||
defer fc.filerAddressesMu.Unlock()
|
||||
|
||||
changed := false
|
||||
filers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)+1)
|
||||
for _, f := range fc.filerAddresses {
|
||||
if f != addr && filerKey(f) == filerKey(addr) {
|
||||
f = addr
|
||||
changed = true
|
||||
}
|
||||
filers[f] = struct{}{}
|
||||
}
|
||||
|
||||
changed := false
|
||||
if update.IsAdd {
|
||||
// A rejoin cancels its deferred leave; bumping the generation keeps an
|
||||
// in-flight snapshot that lacks the rejoined filer from pruning it.
|
||||
@@ -461,45 +465,64 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time.
|
||||
fc.applyDiscoveredFilersLocked(filers)
|
||||
}
|
||||
|
||||
// filerKey identifies a filer regardless of whether its address carries the
|
||||
// gRPC port explicitly (host:port.grpcPort) or implies it (host:port).
|
||||
func filerKey(addr pb.ServerAddress) string {
|
||||
return addr.ToGrpcAddress()
|
||||
}
|
||||
|
||||
func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.ServerAddress]struct{}) {
|
||||
existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses))
|
||||
for _, f := range fc.filerAddresses {
|
||||
existingFilers[f] = struct{}{}
|
||||
discovered := make(map[string]pb.ServerAddress, len(discoveredFilers))
|
||||
for addr := range discoveredFilers {
|
||||
k := filerKey(addr)
|
||||
if prev, dup := discovered[k]; !dup || addr > prev {
|
||||
discovered[k] = addr
|
||||
}
|
||||
}
|
||||
|
||||
// Survivors take the discovered address form and keep their health counters.
|
||||
changed := false
|
||||
kept := make(map[string]struct{}, len(fc.filerAddresses))
|
||||
var removedFilers []pb.ServerAddress
|
||||
newAddresses := make([]pb.ServerAddress, 0, len(discovered))
|
||||
newHealth := make([]*filerHealth, 0, len(discovered))
|
||||
for i, f := range fc.filerAddresses {
|
||||
k := filerKey(f)
|
||||
addr, found := discovered[k]
|
||||
if !found {
|
||||
removedFilers = append(removedFilers, f)
|
||||
continue
|
||||
}
|
||||
if _, dup := kept[k]; dup {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
kept[k] = struct{}{}
|
||||
if addr != f {
|
||||
changed = true
|
||||
}
|
||||
newAddresses = append(newAddresses, addr)
|
||||
newHealth = append(newHealth, fc.filerHealth[i])
|
||||
}
|
||||
|
||||
var newFilers []pb.ServerAddress
|
||||
for addr := range discoveredFilers {
|
||||
if _, found := existingFilers[addr]; !found {
|
||||
for k, addr := range discovered {
|
||||
if _, found := kept[k]; !found {
|
||||
newFilers = append(newFilers, addr)
|
||||
}
|
||||
}
|
||||
|
||||
var removedFilers []pb.ServerAddress
|
||||
for _, f := range fc.filerAddresses {
|
||||
if _, found := discoveredFilers[f]; !found {
|
||||
removedFilers = append(removedFilers, f)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newFilers) == 0 && len(removedFilers) == 0 {
|
||||
if !changed && len(newFilers) == 0 && len(removedFilers) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Remember the active filer so the round-robin pointer can follow it across the rebuild.
|
||||
currentIndex := atomic.LoadInt32(&fc.filerIndex)
|
||||
var currentFiler pb.ServerAddress
|
||||
var currentKey string
|
||||
if currentIndex >= 0 && currentIndex < int32(len(fc.filerAddresses)) {
|
||||
currentFiler = fc.filerAddresses[currentIndex]
|
||||
currentKey = filerKey(fc.filerAddresses[currentIndex])
|
||||
}
|
||||
|
||||
newAddresses := make([]pb.ServerAddress, 0, len(fc.filerAddresses)-len(removedFilers)+len(newFilers))
|
||||
newHealth := make([]*filerHealth, 0, cap(newAddresses))
|
||||
for i, f := range fc.filerAddresses {
|
||||
if _, found := discoveredFilers[f]; found {
|
||||
newAddresses = append(newAddresses, f)
|
||||
newHealth = append(newHealth, fc.filerHealth[i])
|
||||
}
|
||||
}
|
||||
for _, f := range newFilers {
|
||||
newAddresses = append(newAddresses, f)
|
||||
newHealth = append(newHealth, &filerHealth{})
|
||||
@@ -509,9 +532,9 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve
|
||||
fc.filerHealth = newHealth
|
||||
|
||||
var newIndex int32
|
||||
if currentFiler != "" {
|
||||
if currentKey != "" {
|
||||
for i, f := range newAddresses {
|
||||
if f == currentFiler {
|
||||
if filerKey(f) == currentKey {
|
||||
newIndex = int32(i)
|
||||
break
|
||||
}
|
||||
|
||||
@@ -112,6 +112,86 @@ func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDiscoveredFilersKeepsFilerListedWithGrpcPort(t *testing.T) {
|
||||
local := pb.ServerAddress("10.0.0.1:8888")
|
||||
registered := pb.ServerAddress("10.0.0.1:8888.18888")
|
||||
peers := []pb.ServerAddress{"10.0.0.2:8888.18888", "10.0.0.3:8888.18888"}
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
fc := newTestFilerClient(local)
|
||||
health := fc.filerHealth[0]
|
||||
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
|
||||
registered: {},
|
||||
peers[0]: {},
|
||||
peers[1]: {},
|
||||
})
|
||||
|
||||
if got := fc.GetCurrentFiler(); got != registered {
|
||||
t.Fatalf("current filer = %s, want %s", got, registered)
|
||||
}
|
||||
if fc.filerHealth[0] != health {
|
||||
t.Fatalf("local filer lost its health entry")
|
||||
}
|
||||
if got := filerAddressList(fc); len(got) != 3 {
|
||||
t.Fatalf("expected 3 filers, got %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDiscoveredFilersAdoptsAddressForm(t *testing.T) {
|
||||
a := pb.ServerAddress("10.0.0.1:8888")
|
||||
b := pb.ServerAddress("10.0.0.2:8888.18888")
|
||||
|
||||
fc := newTestFilerClient(a, b)
|
||||
atomic.StoreInt32(&fc.filerIndex, 1)
|
||||
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
|
||||
"10.0.0.1:8888.18888": {},
|
||||
b: {},
|
||||
})
|
||||
|
||||
got := filerAddressList(fc)
|
||||
if len(got) != 2 || got[0] != "10.0.0.1:8888.18888" || got[1] != b {
|
||||
t.Fatalf("expected [10.0.0.1:8888.18888 %s], got %v", b, got)
|
||||
}
|
||||
if idx := atomic.LoadInt32(&fc.filerIndex); idx != 1 {
|
||||
t.Errorf("filerIndex = %d, want 1", idx)
|
||||
}
|
||||
fc.RecordFilerFailure(a)
|
||||
if atomic.LoadInt32(&fc.filerHealth[0].failureCount) != 1 {
|
||||
t.Errorf("failure recorded under the old address form was dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDiscoveredFilersPrunesFilerWithOtherGrpcPort(t *testing.T) {
|
||||
old := pb.ServerAddress("10.0.0.1:8888")
|
||||
moved := pb.ServerAddress("10.0.0.1:8888.19999")
|
||||
|
||||
fc := newTestFilerClient(old)
|
||||
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{moved: {}})
|
||||
|
||||
if got := filerAddressList(fc); len(got) != 1 || got[0] != moved {
|
||||
t.Fatalf("expected [%s], got %v", moved, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnPeerUpdateMatchesFilerListedWithGrpcPort(t *testing.T) {
|
||||
local := pb.ServerAddress("10.0.0.1:8888")
|
||||
registered := pb.ServerAddress("10.0.0.1:8888.18888")
|
||||
peer := pb.ServerAddress("10.0.0.2:8888.18888")
|
||||
|
||||
fc := newTestFilerClient(local)
|
||||
fc.OnPeerUpdate(filerUpdate(registered, true), time.Now())
|
||||
if got := filerAddressList(fc); len(got) != 1 || got[0] != registered {
|
||||
t.Fatalf("expected [%s] after the join, got %v", registered, got)
|
||||
}
|
||||
|
||||
fc.OnPeerUpdate(filerUpdate(peer, true), time.Now())
|
||||
fc.OnPeerUpdate(filerUpdate(registered, false), time.Now())
|
||||
if got := filerAddressList(fc); len(got) != 1 || got[0] != peer {
|
||||
t.Fatalf("expected [%s] after the leave, got %v", peer, got)
|
||||
}
|
||||
}
|
||||
|
||||
func filerUpdate(addr pb.ServerAddress, isAdd bool) *master_pb.ClusterNodeUpdate {
|
||||
return &master_pb.ClusterNodeUpdate{NodeType: cluster.FilerType, Address: string(addr), IsAdd: isAdd}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user