mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-08 04:57:50 +01:00
fix: recover TOCTOU + WAL pressure edge case tests
Fix recover path TOCTOU: re-Lookup after AddReplica so the primary refresh assignment includes the freshly added replica addresses. Previously, Lookup (copy) was called before AddReplica modified the registry, so entry.Replicas was empty → primary got replicas=0 → shipper never configured. Add 2 WAL pressure edge case tests: - ShipperCatchUpOrEscalate: 64KB WAL, 200 writes, aggressive flusher. Proves no hang/deadlock/corruption. Shipper either keeps up or correctly escalates to NeedsRebuild. - RebuildWithPinWhilePrimaryWrites: rebuild session active while primary writes 7600+ blocks in 2s. Proves primary never freezes — rebuild pin is on replica only, primary WAL recycles freely. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e0116fc631
commit
53246d2780
3 files changed
+285
-5
No files matched your search
@@ -561,12 +561,16 @@ func (ms *MasterServer) recoverBlockVolumes(reconnectedServer string) {
|
||||
CtrlAddr: ctrlAddr,
|
||||
})
|
||||
|
||||
// Re-lookup entry after AddReplica so the entry copy reflects the
|
||||
// updated replica list. Without this, the stale copy from line 540
|
||||
// has 0 replicas, and the primary refresh assignment gets empty
|
||||
// ReplicaAddrs — the primary never configures its shipper.
|
||||
entry, ok = ms.blockRegistry.Lookup(rb.VolumeName)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// CP13-8: Try catch-up first (Replica assignment), fall back to rebuild.
|
||||
// If the replica can catch up from the primary's retained WAL, this is
|
||||
// much faster than a full rebuild. The shipper's reconnect handshake
|
||||
// (CP13-5) determines whether catch-up or rebuild is actually needed.
|
||||
// If catch-up fails, the shipper marks NeedsRebuild, and the master
|
||||
// sends a Rebuilding assignment on the next heartbeat cycle.
|
||||
if dataAddr != "" {
|
||||
leaseTTLMs := blockvol.LeaseTTLToWire(30 * time.Second)
|
||||
// Send Replica assignment to the reconnected server.
|
||||
|
||||
@@ -314,6 +314,13 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
|
||||
// Pending assignments (role changes, epoch bumps).
|
||||
pending := ms.blockAssignmentQueue.Peek(dn.Url())
|
||||
if len(pending) > 0 {
|
||||
glog.V(0).Infof("block assign: delivering %d assignment(s) to %s", len(pending), dn.Url())
|
||||
for _, p := range pending {
|
||||
glog.V(0).Infof(" assign: path=%s epoch=%d role=%d replicas=%d",
|
||||
p.Path, p.Epoch, p.Role, len(p.ReplicaAddrs))
|
||||
}
|
||||
}
|
||||
|
||||
// Lease grants for confirmed volumes (lightweight: path+epoch+ttl).
|
||||
// Skip volumes that already have a pending assignment.
|
||||
|
||||
Reference in new issue
Block a user