From 05a1463855e323cf9591a3a2523bcebde483528c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 19 Nov 2026 12:42:31 -0800 Subject: [PATCH] wdclient: keep the embedded S3 gateway on its local filer across discovery (#12771) * filer: keep an explicit gRPC port in the embedded gateways' filer address runFiler handed the embedded S3/WebDAV/IAM/SFTP gateways NewServerAddress(...).String(), and String() drops the .grpcPort part. With a non-default -port.grpc the gateways then dialed port+10000 and did not recognize their own filer in the master's list. Co-Authored-By: Claude Opus 5.5 * wdclient: match discovered filers by gRPC address A filer listed as host:port and the same filer discovered as host:port.grpcPort were compared as raw strings, so a refresh pruned the configured filer, added every discovered filer as new in map order, and reset the current filer to whichever came first; a pushed join added it a second time. Key filers by their gRPC address: a survivor keeps its health and current position and takes the discovered address form, and pushed joins and leaves match the same way. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/command/filer.go | 2 +- weed/wdclient/filer_client.go | 83 +++++++++++++++++++----------- weed/wdclient/filer_client_test.go | 80 ++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 8931abaafa..2244db3629 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -307,7 +307,7 @@ func runFiler(cmd *Command, args []string) bool { // -ip.bind is unset and the bind address falls back to -ip. This must be // captured before startFiler fills *f.bindIp in place. filerBindIpUnset := *f.bindIp == "" - filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String() + filerAddress := string(pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc)) startDelay := time.Duration(2) if *filerStartS3 { filerS3Options.filer = &filerAddress diff --git a/weed/wdclient/filer_client.go b/weed/wdclient/filer_client.go index 7f6f8df94f..b43fd3f09e 100644 --- a/weed/wdclient/filer_client.go +++ b/weed/wdclient/filer_client.go @@ -230,7 +230,7 @@ func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) { // Find the index of the specified filer address for i, filer := range fc.filerAddresses { - if filer == addr { + if filerKey(filer) == filerKey(addr) { atomic.StoreInt32(&fc.filerIndex, int32(i)) return } @@ -246,7 +246,7 @@ func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool { // Find the health for this filer address for i, filer := range fc.filerAddresses { - if filer == addr { + if filerKey(filer) == filerKey(addr) { if i < len(fc.filerHealth) { return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i]) } @@ -264,7 +264,7 @@ func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) { // Find the health for this filer address for i, filer := range fc.filerAddresses { - if filer == addr { + if filerKey(filer) == filerKey(addr) { if i < len(fc.filerHealth) { fc.recordFilerSuccessWithHealth(fc.filerHealth[i]) } @@ -280,7 +280,7 @@ func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) { // Find the health for this filer address for i, filer := range fc.filerAddresses { - if filer == addr { + if filerKey(filer) == filerKey(addr) { if i < len(fc.filerHealth) { fc.recordFilerFailureWithHealth(fc.filerHealth[i]) } @@ -413,12 +413,16 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time. fc.filerAddressesMu.Lock() defer fc.filerAddressesMu.Unlock() + changed := false filers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)+1) for _, f := range fc.filerAddresses { + if f != addr && filerKey(f) == filerKey(addr) { + f = addr + changed = true + } filers[f] = struct{}{} } - changed := false if update.IsAdd { // A rejoin cancels its deferred leave; bumping the generation keeps an // in-flight snapshot that lacks the rejoined filer from pruning it. @@ -461,45 +465,64 @@ func (fc *FilerClient) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, _ time. fc.applyDiscoveredFilersLocked(filers) } +// filerKey identifies a filer regardless of whether its address carries the +// gRPC port explicitly (host:port.grpcPort) or implies it (host:port). +func filerKey(addr pb.ServerAddress) string { + return addr.ToGrpcAddress() +} + func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.ServerAddress]struct{}) { - existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)) - for _, f := range fc.filerAddresses { - existingFilers[f] = struct{}{} + discovered := make(map[string]pb.ServerAddress, len(discoveredFilers)) + for addr := range discoveredFilers { + k := filerKey(addr) + if prev, dup := discovered[k]; !dup || addr > prev { + discovered[k] = addr + } + } + + // Survivors take the discovered address form and keep their health counters. + changed := false + kept := make(map[string]struct{}, len(fc.filerAddresses)) + var removedFilers []pb.ServerAddress + newAddresses := make([]pb.ServerAddress, 0, len(discovered)) + newHealth := make([]*filerHealth, 0, len(discovered)) + for i, f := range fc.filerAddresses { + k := filerKey(f) + addr, found := discovered[k] + if !found { + removedFilers = append(removedFilers, f) + continue + } + if _, dup := kept[k]; dup { + changed = true + continue + } + kept[k] = struct{}{} + if addr != f { + changed = true + } + newAddresses = append(newAddresses, addr) + newHealth = append(newHealth, fc.filerHealth[i]) } var newFilers []pb.ServerAddress - for addr := range discoveredFilers { - if _, found := existingFilers[addr]; !found { + for k, addr := range discovered { + if _, found := kept[k]; !found { newFilers = append(newFilers, addr) } } - var removedFilers []pb.ServerAddress - for _, f := range fc.filerAddresses { - if _, found := discoveredFilers[f]; !found { - removedFilers = append(removedFilers, f) - } - } - - if len(newFilers) == 0 && len(removedFilers) == 0 { + if !changed && len(newFilers) == 0 && len(removedFilers) == 0 { return } // Remember the active filer so the round-robin pointer can follow it across the rebuild. currentIndex := atomic.LoadInt32(&fc.filerIndex) - var currentFiler pb.ServerAddress + var currentKey string if currentIndex >= 0 && currentIndex < int32(len(fc.filerAddresses)) { - currentFiler = fc.filerAddresses[currentIndex] + currentKey = filerKey(fc.filerAddresses[currentIndex]) } - newAddresses := make([]pb.ServerAddress, 0, len(fc.filerAddresses)-len(removedFilers)+len(newFilers)) - newHealth := make([]*filerHealth, 0, cap(newAddresses)) - for i, f := range fc.filerAddresses { - if _, found := discoveredFilers[f]; found { - newAddresses = append(newAddresses, f) - newHealth = append(newHealth, fc.filerHealth[i]) - } - } for _, f := range newFilers { newAddresses = append(newAddresses, f) newHealth = append(newHealth, &filerHealth{}) @@ -509,9 +532,9 @@ func (fc *FilerClient) applyDiscoveredFilersLocked(discoveredFilers map[pb.Serve fc.filerHealth = newHealth var newIndex int32 - if currentFiler != "" { + if currentKey != "" { for i, f := range newAddresses { - if f == currentFiler { + if filerKey(f) == currentKey { newIndex = int32(i) break } diff --git a/weed/wdclient/filer_client_test.go b/weed/wdclient/filer_client_test.go index a66d88efa2..1ff2256e29 100644 --- a/weed/wdclient/filer_client_test.go +++ b/weed/wdclient/filer_client_test.go @@ -112,6 +112,86 @@ func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) { } } +func TestApplyDiscoveredFilersKeepsFilerListedWithGrpcPort(t *testing.T) { + local := pb.ServerAddress("10.0.0.1:8888") + registered := pb.ServerAddress("10.0.0.1:8888.18888") + peers := []pb.ServerAddress{"10.0.0.2:8888.18888", "10.0.0.3:8888.18888"} + + for i := 0; i < 200; i++ { + fc := newTestFilerClient(local) + health := fc.filerHealth[0] + fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{ + registered: {}, + peers[0]: {}, + peers[1]: {}, + }) + + if got := fc.GetCurrentFiler(); got != registered { + t.Fatalf("current filer = %s, want %s", got, registered) + } + if fc.filerHealth[0] != health { + t.Fatalf("local filer lost its health entry") + } + if got := filerAddressList(fc); len(got) != 3 { + t.Fatalf("expected 3 filers, got %v", got) + } + } +} + +func TestApplyDiscoveredFilersAdoptsAddressForm(t *testing.T) { + a := pb.ServerAddress("10.0.0.1:8888") + b := pb.ServerAddress("10.0.0.2:8888.18888") + + fc := newTestFilerClient(a, b) + atomic.StoreInt32(&fc.filerIndex, 1) + fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{ + "10.0.0.1:8888.18888": {}, + b: {}, + }) + + got := filerAddressList(fc) + if len(got) != 2 || got[0] != "10.0.0.1:8888.18888" || got[1] != b { + t.Fatalf("expected [10.0.0.1:8888.18888 %s], got %v", b, got) + } + if idx := atomic.LoadInt32(&fc.filerIndex); idx != 1 { + t.Errorf("filerIndex = %d, want 1", idx) + } + fc.RecordFilerFailure(a) + if atomic.LoadInt32(&fc.filerHealth[0].failureCount) != 1 { + t.Errorf("failure recorded under the old address form was dropped") + } +} + +func TestApplyDiscoveredFilersPrunesFilerWithOtherGrpcPort(t *testing.T) { + old := pb.ServerAddress("10.0.0.1:8888") + moved := pb.ServerAddress("10.0.0.1:8888.19999") + + fc := newTestFilerClient(old) + fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{moved: {}}) + + if got := filerAddressList(fc); len(got) != 1 || got[0] != moved { + t.Fatalf("expected [%s], got %v", moved, got) + } +} + +func TestOnPeerUpdateMatchesFilerListedWithGrpcPort(t *testing.T) { + local := pb.ServerAddress("10.0.0.1:8888") + registered := pb.ServerAddress("10.0.0.1:8888.18888") + peer := pb.ServerAddress("10.0.0.2:8888.18888") + + fc := newTestFilerClient(local) + fc.OnPeerUpdate(filerUpdate(registered, true), time.Now()) + if got := filerAddressList(fc); len(got) != 1 || got[0] != registered { + t.Fatalf("expected [%s] after the join, got %v", registered, got) + } + + fc.OnPeerUpdate(filerUpdate(peer, true), time.Now()) + fc.OnPeerUpdate(filerUpdate(registered, false), time.Now()) + if got := filerAddressList(fc); len(got) != 1 || got[0] != peer { + t.Fatalf("expected [%s] after the leave, got %v", peer, got) + } +} + func filerUpdate(addr pb.ServerAddress, isAdd bool) *master_pb.ClusterNodeUpdate { return &master_pb.ClusterNodeUpdate{NodeType: cluster.FilerType, Address: string(addr), IsAdd: isAdd} }