mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 01:37:10 +01:00
refactor: close bounded recovery drain and invalidation seams
Move removed-replica drain and replica-scoped invalidation onto explicit core-command paths so the widened multi-replica runtime no longer depends on coarse host-side recovery handling. Made-with: Cursor
This commit is contained in:
1 parent
5fd9ec0edf
commit
43dbebfa04
12 files changed
+751
-151
No files matched your search
@@ -14,7 +14,8 @@ type fakeOps struct {
|
||||
startReceiverFn func(blockvol.BlockVolumeAssignment) (bool, error)
|
||||
configureShipperFn func(string, []engine.ReplicaAssignment) (bool, bool, error)
|
||||
startRecoveryTaskFn func(string, blockvol.BlockVolumeAssignment) (bool, error)
|
||||
invalidateSessionFn func(string, string) (bool, error)
|
||||
drainRecoveryTaskFn func(string, string) (bool, error)
|
||||
invalidateSessionFn func(string, string, string) (bool, error)
|
||||
startCatchUpFn func(string, uint64) (bool, error)
|
||||
startRebuildFn func(string, uint64) (bool, error)
|
||||
}
|
||||
@@ -47,11 +48,18 @@ func (f fakeOps) StartRecoveryTask(replicaID string, assignment blockvol.BlockVo
|
||||
return f.startRecoveryTaskFn(replicaID, assignment)
|
||||
}
|
||||
|
||||
func (f fakeOps) InvalidateSession(volumeID, reason string) (bool, error) {
|
||||
func (f fakeOps) DrainRecoveryTask(replicaID, reason string) (bool, error) {
|
||||
if f.drainRecoveryTaskFn == nil {
|
||||
return false, nil
|
||||
}
|
||||
return f.drainRecoveryTaskFn(replicaID, reason)
|
||||
}
|
||||
|
||||
func (f fakeOps) InvalidateSession(volumeID, replicaID, reason string) (bool, error) {
|
||||
if f.invalidateSessionFn == nil {
|
||||
return false, nil
|
||||
}
|
||||
return f.invalidateSessionFn(volumeID, reason)
|
||||
return f.invalidateSessionFn(volumeID, replicaID, reason)
|
||||
}
|
||||
|
||||
func (f fakeOps) StartCatchUp(replicaID string, targetLSN uint64) (bool, error) {
|
||||
@@ -188,7 +196,11 @@ func TestDispatcher_StopsOnFirstError(t *testing.T) {
|
||||
type fakeRecoveryCoordinator struct {
|
||||
startedReplica string
|
||||
startedAssigns []blockvol.BlockVolumeAssignment
|
||||
catchUpCalls []struct {
|
||||
drained []struct {
|
||||
replicaID string
|
||||
reason string
|
||||
}
|
||||
catchUpCalls []struct {
|
||||
replicaID string
|
||||
targetLSN uint64
|
||||
}
|
||||
@@ -203,6 +215,13 @@ func (f *fakeRecoveryCoordinator) StartRecoveryTask(replicaID string, assignment
|
||||
f.startedAssigns = assignments
|
||||
}
|
||||
|
||||
func (f *fakeRecoveryCoordinator) DrainRecoveryTask(replicaID, reason string) {
|
||||
f.drained = append(f.drained, struct {
|
||||
replicaID string
|
||||
reason string
|
||||
}{replicaID: replicaID, reason: reason})
|
||||
}
|
||||
|
||||
func (f *fakeRecoveryCoordinator) ExecutePendingCatchUp(replicaID string, targetLSN uint64) error {
|
||||
f.catchUpCalls = append(f.catchUpCalls, struct {
|
||||
replicaID string
|
||||
@@ -268,6 +287,29 @@ func TestServiceOps_StartRecoveryTaskUsesRecoveryCoordinator(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatcher_DrainRecoveryTaskRecordsExecution(t *testing.T) {
|
||||
effects := &fakeEffects{}
|
||||
var drained []string
|
||||
d := NewDispatcher(fakeOps{
|
||||
drainRecoveryTaskFn: func(replicaID, reason string) (bool, error) {
|
||||
drained = append(drained, replicaID+":"+reason)
|
||||
return true, nil
|
||||
},
|
||||
}, effects)
|
||||
err := d.Run([]engine.Command{
|
||||
engine.DrainRecoveryTaskCommand{VolumeID: "vol1", ReplicaID: "vol1/vs2", Reason: "assignment_removed"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(drained, []string{"vol1/vs2:assignment_removed"}) {
|
||||
t.Fatalf("drained=%v", drained)
|
||||
}
|
||||
if !reflect.DeepEqual(effects.recorded, []string{"vol1:drain_recovery_task"}) {
|
||||
t.Fatalf("recorded=%v", effects.recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostEffects_PublishProjectionPrefersLatestCoreProjection(t *testing.T) {
|
||||
cache := &fakeProjectionCache{}
|
||||
effects := NewHostEffects(
|
||||
@@ -342,7 +384,7 @@ func TestServiceOps_InvalidateSessionUsesProjectionAndSenderResolver(t *testing.
|
||||
}
|
||||
},
|
||||
)
|
||||
executed, err := ops.InvalidateSession("vol1", "test_reason")
|
||||
executed, err := ops.InvalidateSession("vol1", "", "test_reason")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -357,6 +399,60 @@ func TestServiceOps_InvalidateSessionUsesProjectionAndSenderResolver(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceOps_InvalidateSessionTargetsSingleReplicaWhenProvided(t *testing.T) {
|
||||
s1 := &fakeSessionInvalidator{}
|
||||
s2 := &fakeSessionInvalidator{}
|
||||
ops := NewServiceOps(
|
||||
fakeOps{},
|
||||
nil,
|
||||
fakeProjectionReader{
|
||||
ok: true,
|
||||
proj: engine.PublicationProjection{
|
||||
VolumeID: "vol1",
|
||||
ReplicaIDs: []string{"vol1/vs2", "vol1/vs3"},
|
||||
},
|
||||
},
|
||||
func(replicaID string) SessionInvalidator {
|
||||
switch replicaID {
|
||||
case "vol1/vs2":
|
||||
return s1
|
||||
case "vol1/vs3":
|
||||
return s2
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
},
|
||||
)
|
||||
executed, err := ops.InvalidateSession("vol1", "vol1/vs2", "test_reason")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !executed {
|
||||
t.Fatal("expected executed")
|
||||
}
|
||||
if !reflect.DeepEqual(s1.reasons, []string{"test_reason"}) {
|
||||
t.Fatalf("reasons1=%v", s1.reasons)
|
||||
}
|
||||
if len(s2.reasons) != 0 {
|
||||
t.Fatalf("reasons2=%v", s2.reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceOps_DrainRecoveryTaskUsesRecoveryCoordinator(t *testing.T) {
|
||||
rec := &fakeRecoveryCoordinator{}
|
||||
ops := NewServiceOps(fakeOps{}, rec, nil, nil)
|
||||
executed, err := ops.DrainRecoveryTask("vol1/vs2", "assignment_removed")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !executed {
|
||||
t.Fatal("expected executed")
|
||||
}
|
||||
if len(rec.drained) != 1 || rec.drained[0].replicaID != "vol1/vs2" || rec.drained[0].reason != "assignment_removed" {
|
||||
t.Fatalf("drained=%v", rec.drained)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceOps_StartRecoveryTask_NilRecoveryIsNoop(t *testing.T) {
|
||||
ops := NewServiceOps(fakeOps{}, nil, nil, nil)
|
||||
executed, err := ops.StartRecoveryTask("vol1/vs2", blockvol.BlockVolumeAssignment{Path: "vol1"})
|
||||
|
||||
Reference in new issue
Block a user