mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
* filer: stop following a peer that stays unreachable * filer: re-track an answering peer only while it is still followed A signature read that succeeds just before the master removes the peer called initPeerWatermark after the removal, clearing its mark. The loop then exits, so the watermark it re-tracked never advances or drops and pins aggregated readers at the settled horizon. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: forget a peer's unreachable clock when the master removes it Only a successful connection cleared the clock, so every departed address that once failed stayed in the map for the filer's lifetime, and a re-added peer inherited the old outage. The removal now drops it, and the clock is guarded by peerChansLock so the removed peer's exiting loop cannot start a new one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: report the unreachable mark only when it is set Once peerResumeRetention forgets the mark of a peer that stays unreachable, markPeerWatermarkRemoved no longer marks it (its watermarks are gone), but notePeerUnreachable still saw no mark and logged "treating it as removed" on every retry. markPeerWatermarkRemoved now reports whether it set a new mark. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: fix the unreachable-peer subscriber's start before the baseline write The subscriber goroutine picked its start time itself, so a baseline write that finished first fell before the subscription and the test failed after 20 seconds with correct delivery. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
4.1 KiB
Go
117 lines
4.1 KiB
Go
//go:build !windows
|
|
|
|
package metadata_subscribe
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/seaweedfs/seaweedfs/test/testutil"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
// A peer removal is broadcast only to the filers connected to the master that
|
|
// saw the peer go. A peer that dies while no master is up (or across a leader
|
|
// change) is never removed: the filer keeps retrying its address, the peer's
|
|
// delivery watermark stays at zero, and every aggregated subscriber of that
|
|
// filer is held at the settled horizon (2 x LogFlushInterval) for good.
|
|
func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skipping integration test in short mode")
|
|
}
|
|
|
|
weedBinary := findWeedBinary()
|
|
require.NotEmpty(t, weedBinary, "weed binary not found")
|
|
|
|
testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_")
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
if t.Failed() {
|
|
t.Logf("logs kept at %s", testDir)
|
|
return
|
|
}
|
|
os.RemoveAll(testDir)
|
|
})
|
|
|
|
ports, err := testutil.AllocateMiniPorts(3)
|
|
require.NoError(t, err)
|
|
masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
|
|
master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
|
|
filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
|
|
|
|
masterArgs := []string{"master",
|
|
"-ip=127.0.0.1",
|
|
"-port=" + strconv.Itoa(masterPort),
|
|
"-mdir=" + mkdir(t, testDir, "master"),
|
|
"-peers=none"}
|
|
masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
|
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
|
startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
|
|
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
|
|
filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
|
|
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
|
|
filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
|
|
|
|
// filer1 follows filer2: an entry created on filer2 reaches filer1
|
|
createPeerEntry(t, filer2Address, "from-filer2")
|
|
require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second),
|
|
"filer1 never followed filer2")
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
events := make(chan *filer_pb.SubscribeMetadataResponse, 1000)
|
|
sinceNs := time.Now().UnixNano()
|
|
go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events)
|
|
arrived := make(map[string]time.Time)
|
|
waitArrival := func(name string, within time.Duration) bool {
|
|
deadline := time.After(within)
|
|
for {
|
|
if _, ok := arrived[name]; ok {
|
|
return true
|
|
}
|
|
select {
|
|
case ev := <-events:
|
|
if n := ev.GetEventNotification().GetNewEntry(); n != nil {
|
|
arrived[n.Name] = time.Now()
|
|
}
|
|
case <-deadline:
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
createPeerEntry(t, filer1Address, "baseline")
|
|
require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber")
|
|
|
|
// filer2 dies while no master is up, so no removal is ever broadcast
|
|
stopProcess(masterProcess)
|
|
stopProcess(filer2)
|
|
startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
|
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
|
require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second))
|
|
diedAt := time.Now()
|
|
|
|
// Unpatched, every event from here on arrives 2 x LogFlushInterval late.
|
|
// The dead peer may hold reads for its unreachable bound plus the removal
|
|
// grace (2 x LogFlushInterval each); after that delivery must be prompt.
|
|
for i := 0; ; i++ {
|
|
name := fmt.Sprintf("probe-%d", i)
|
|
createPeerEntry(t, filer1Address, name)
|
|
if waitArrival(name, 10*time.Second) {
|
|
t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second))
|
|
return
|
|
}
|
|
if time.Since(diedAt) > 6*time.Minute {
|
|
t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second))
|
|
}
|
|
}
|
|
}
|