wdclient: keep the embedded S3 gateway on its local filer across discovery (#12771)

* filer: keep an explicit gRPC port in the embedded gateways' filer address

runFiler handed the embedded S3/WebDAV/IAM/SFTP gateways
NewServerAddress(...).String(), and String() drops the .grpcPort part.
With a non-default -port.grpc the gateways then dialed port+10000 and
did not recognize their own filer in the master's list.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* wdclient: match discovered filers by gRPC address

A filer listed as host:port and the same filer discovered as
host:port.grpcPort were compared as raw strings, so a refresh pruned the
configured filer, added every discovered filer as new in map order, and
reset the current filer to whichever came first; a pushed join added it
a second time. Key filers by their gRPC address: a survivor keeps its
health and current position and takes the discovered address form, and
pushed joins and leaves match the same way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-19 12:42:31 -08:00
1 parent c1c95e127f
commit 05a1463855
3 files changed
+134 -31

No files matched your search

+1 -1
View File
@@ -307,7 +307,7 @@ func runFiler(cmd *Command, args []string) bool {
// -ip.bind is unset and the bind address falls back to -ip. This must be
// captured before startFiler fills *f.bindIp in place.
filerBindIpUnset := *f.bindIp == ""
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
filerAddress := string(pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc))
startDelay := time.Duration(2)
if *filerStartS3 {
filerS3Options.filer = &filerAddress
+53 -30
View File
@@ -230,7 +230,7 @@ func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) {
// Find the index of the specified filer address
for i, filer := range fc.filerAddresses {
if filer == addr {
if filerKey(filer) == filerKey(addr) {
atomic.StoreInt32(&fc.filerIndex, int32(i))
return
}
@@ -246,7 +246,7 @@ func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool {
// Find the health for this filer address
for i, filer := range fc.filerAddresses {
if filer == addr {
if filerKey(filer) == filerKey(addr) {
if i < len(fc.filerHealth) {
return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i])
}
@@ -264,7 +264,7 @@ func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) {
// Find the health for this filer address
for i, filer := range fc.filerAddresses {
if filer == addr {
if filerKey(filer) == filerKey(addr) {
if i < len(fc.filerHealth) {
fc.recordFilerSuccessWithHealth(fc.filerHealth[i])
}
@@ -280,7 +280,7 @@ func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) {
// Find the health for this filer address
for i, filer := range fc.filerAddresses {
if filer == addr {
if filerKey(filer) == filerKey(addr) {
if i < len(fc.filerHealth) {
fc.recordFilerFailureWithHealth(fc.filerHealth[i])
}
@@ -413,12 +413,16 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time.
fc.filerAddressesMu.Lock()
defer fc.filerAddressesMu.Unlock()
changed := false
filers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)+1)
for _, f := range fc.filerAddresses {
if f != addr && filerKey(f) == filerKey(addr) {
f = addr
changed = true
}
filers[f] = struct{}{}
}
changed := false
if update.IsAdd {
// A rejoin cancels its deferred leave; bumping the generation keeps an
// in-flight snapshot that lacks the rejoined filer from pruning it.
@@ -461,45 +465,64 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time.
fc.applyDiscoveredFilersLocked(filers)
}
// filerKey identifies a filer regardless of whether its address carries the
// gRPC port explicitly (host:port.grpcPort) or implies it (host:port).
func filerKey(addr pb.ServerAddress) string {
return addr.ToGrpcAddress()
}
func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.ServerAddress]struct{}) {
existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses))
for _, f := range fc.filerAddresses {
existingFilers[f] = struct{}{}
discovered := make(map[string]pb.ServerAddress, len(discoveredFilers))
for addr := range discoveredFilers {
k := filerKey(addr)
if prev, dup := discovered[k]; !dup || addr > prev {
discovered[k] = addr
}
}
// Survivors take the discovered address form and keep their health counters.
changed := false
kept := make(map[string]struct{}, len(fc.filerAddresses))
var removedFilers []pb.ServerAddress
newAddresses := make([]pb.ServerAddress, 0, len(discovered))
newHealth := make([]*filerHealth, 0, len(discovered))
for i, f := range fc.filerAddresses {
k := filerKey(f)
addr, found := discovered[k]
if !found {
removedFilers = append(removedFilers, f)
continue
}
if _, dup := kept[k]; dup {
changed = true
continue
}
kept[k] = struct{}{}
if addr != f {
changed = true
}
newAddresses = append(newAddresses, addr)
newHealth = append(newHealth, fc.filerHealth[i])
}
var newFilers []pb.ServerAddress
for addr := range discoveredFilers {
if _, found := existingFilers[addr]; !found {
for k, addr := range discovered {
if _, found := kept[k]; !found {
newFilers = append(newFilers, addr)
}
}
var removedFilers []pb.ServerAddress
for _, f := range fc.filerAddresses {
if _, found := discoveredFilers[f]; !found {
removedFilers = append(removedFilers, f)
}
}
if len(newFilers) == 0 && len(removedFilers) == 0 {
if !changed && len(newFilers) == 0 && len(removedFilers) == 0 {
return
}
// Remember the active filer so the round-robin pointer can follow it across the rebuild.
currentIndex := atomic.LoadInt32(&fc.filerIndex)
var currentFiler pb.ServerAddress
var currentKey string
if currentIndex >= 0 && currentIndex < int32(len(fc.filerAddresses)) {
currentFiler = fc.filerAddresses[currentIndex]
currentKey = filerKey(fc.filerAddresses[currentIndex])
}
newAddresses := make([]pb.ServerAddress, 0, len(fc.filerAddresses)-len(removedFilers)+len(newFilers))
newHealth := make([]*filerHealth, 0, cap(newAddresses))
for i, f := range fc.filerAddresses {
if _, found := discoveredFilers[f]; found {
newAddresses = append(newAddresses, f)
newHealth = append(newHealth, fc.filerHealth[i])
}
}
for _, f := range newFilers {
newAddresses = append(newAddresses, f)
newHealth = append(newHealth, &filerHealth{})
@@ -509,9 +532,9 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve
fc.filerHealth = newHealth
var newIndex int32
if currentFiler != "" {
if currentKey != "" {
for i, f := range newAddresses {
if f == currentFiler {
if filerKey(f) == currentKey {
newIndex = int32(i)
break
}
+80
View File
@@ -112,6 +112,86 @@ func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) {
}
}
func TestApplyDiscoveredFilersKeepsFilerListedWithGrpcPort(t *testing.T) {
local := pb.ServerAddress("10.0.0.1:8888")
registered := pb.ServerAddress("10.0.0.1:8888.18888")
peers := []pb.ServerAddress{"10.0.0.2:8888.18888", "10.0.0.3:8888.18888"}
for i := 0; i < 200; i++ {
fc := newTestFilerClient(local)
health := fc.filerHealth[0]
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
registered: {},
peers[0]: {},
peers[1]: {},
})
if got := fc.GetCurrentFiler(); got != registered {
t.Fatalf("current filer = %s, want %s", got, registered)
}
if fc.filerHealth[0] != health {
t.Fatalf("local filer lost its health entry")
}
if got := filerAddressList(fc); len(got) != 3 {
t.Fatalf("expected 3 filers, got %v", got)
}
}
}
func TestApplyDiscoveredFilersAdoptsAddressForm(t *testing.T) {
a := pb.ServerAddress("10.0.0.1:8888")
b := pb.ServerAddress("10.0.0.2:8888.18888")
fc := newTestFilerClient(a, b)
atomic.StoreInt32(&fc.filerIndex, 1)
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
"10.0.0.1:8888.18888": {},
b: {},
})
got := filerAddressList(fc)
if len(got) != 2 || got[0] != "10.0.0.1:8888.18888" || got[1] != b {
t.Fatalf("expected [10.0.0.1:8888.18888 %s], got %v", b, got)
}
if idx := atomic.LoadInt32(&fc.filerIndex); idx != 1 {
t.Errorf("filerIndex = %d, want 1", idx)
}
fc.RecordFilerFailure(a)
if atomic.LoadInt32(&fc.filerHealth[0].failureCount) != 1 {
t.Errorf("failure recorded under the old address form was dropped")
}
}
func TestApplyDiscoveredFilersPrunesFilerWithOtherGrpcPort(t *testing.T) {
old := pb.ServerAddress("10.0.0.1:8888")
moved := pb.ServerAddress("10.0.0.1:8888.19999")
fc := newTestFilerClient(old)
fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{moved: {}})
if got := filerAddressList(fc); len(got) != 1 || got[0] != moved {
t.Fatalf("expected [%s], got %v", moved, got)
}
}
func TestOnPeerUpdateMatchesFilerListedWithGrpcPort(t *testing.T) {
local := pb.ServerAddress("10.0.0.1:8888")
registered := pb.ServerAddress("10.0.0.1:8888.18888")
peer := pb.ServerAddress("10.0.0.2:8888.18888")
fc := newTestFilerClient(local)
fc.OnPeerUpdate(filerUpdate(registered, true), time.Now())
if got := filerAddressList(fc); len(got) != 1 || got[0] != registered {
t.Fatalf("expected [%s] after the join, got %v", registered, got)
}
fc.OnPeerUpdate(filerUpdate(peer, true), time.Now())
fc.OnPeerUpdate(filerUpdate(registered, false), time.Now())
if got := filerAddressList(fc); len(got) != 1 || got[0] != peer {
t.Fatalf("expected [%s] after the leave, got %v", peer, got)
}
}
func filerUpdate(addr pb.ServerAddress, isAdd bool) *master_pb.ClusterNodeUpdate {
return &master_pb.ClusterNodeUpdate{NodeType: cluster.FilerType, Address: string(addr), IsAdd: isAdd}
}