seaweedfs/weed/server/volume_grpc_block_activation_gate_test.go

package weed_server

import (
	"log"
	"os"
	"testing"

	engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi"
)

func TestT4_DegradedProjection_GatesActivation(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-degraded")

	// Inject degraded projection.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "barrier_timeout"},
	}
	bs.coreProjMu.Unlock()

	bs.evaluateActivationGate(path)

	gated, reason := bs.IsActivationGated(path)
	if !gated {
		t.Fatal("expected activation gated for degraded projection")
	}
	if reason == "" {
		t.Fatal("expected non-empty gate reason")
	}
}

func TestT4_NeedsRebuildProjection_GatesActivation(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-rebuild")

	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeNeedsRebuild, Reason: "gap_too_large"},
	}
	bs.coreProjMu.Unlock()

	bs.evaluateActivationGate(path)

	gated, reason := bs.IsActivationGated(path)
	if !gated {
		t.Fatal("expected activation gated for needs_rebuild projection")
	}
	if reason == "" {
		t.Fatal("expected non-empty gate reason")
	}
}

func TestT4_HealthyProjection_ClearsGate(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-healthy")

	// Start gated.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "barrier_timeout"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, _ := bs.IsActivationGated(path); !gated {
		t.Fatal("expected gated initially")
	}

	// Transition to healthy.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModePublishHealthy},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)

	if gated, _ := bs.IsActivationGated(path); gated {
		t.Fatal("expected gate cleared for publish_healthy projection")
	}
}

func TestT4_GateEnforcedBeforeHeartbeat(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-pre-hb")

	// Inject degraded projection and evaluate gate.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "test"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)

	// Gate is set BEFORE any heartbeat. Verify the heartbeat carries
	// the gated state.
	msgs := bs.CollectBlockVolumeHeartbeat()
	var found *blockvol.BlockVolumeInfoMessage
	for i := range msgs {
		if msgs[i].Path == path {
			found = &msgs[i]
			break
		}
	}
	if found == nil {
		t.Fatal("heartbeat message not found for gated volume")
	}
	if !found.ActivationGated {
		t.Fatal("heartbeat should report ActivationGated=true")
	}
	if found.ActivationGateReason == "" {
		t.Fatal("heartbeat should report non-empty ActivationGateReason")
	}
}

func TestT4_RecoveryFromGated_ReenablesServing(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-recovery")

	// Start gated (needs_rebuild).
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeNeedsRebuild, Reason: "gap"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, _ := bs.IsActivationGated(path); !gated {
		t.Fatal("expected gated")
	}

	// Simulate recovery: projection transitions to replica_ready.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeReplicaReady},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, _ := bs.IsActivationGated(path); gated {
		t.Fatal("expected gate cleared after recovery to replica_ready")
	}

	// And then to publish_healthy.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModePublishHealthy},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, _ := bs.IsActivationGated(path); gated {
		t.Fatal("expected gate cleared after recovery to publish_healthy")
	}
}

func TestT4_ApplyCoreAssignment_GatesDegradedPrimary(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-assignment")

	// Process primary assignment through the core path. The V2 core will
	// compute the projection (allocated_only for a no-replica assignment).
	bs.applyCoreAssignmentEvent(blockvol.BlockVolumeAssignment{
		Path:       path,
		Epoch:      5,
		Role:       blockvol.RoleToWire(blockvol.RolePrimary),
		LeaseTtlMs: 30000,
	})

	// Fresh assignment with no replicas → allocated_only. This must NOT
	// be gated (no stale data risk on fresh bootstrap).
	if gated, reason := bs.IsActivationGated(path); gated {
		t.Fatalf("fresh primary assignment must not be gated, got reason=%q", reason)
	}

	// Now inject degraded projection (simulating barrier failure) and
	// re-evaluate. THIS must gate.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		VolumeID: path,
		Mode:     engine.ModeView{Name: engine.ModeDegraded, Reason: "incomplete_reconstruction"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)

	gated, reason := bs.IsActivationGated(path)
	if !gated {
		t.Fatal("expected activation gated after projection transitions to degraded")
	}
	if reason == "" {
		t.Fatal("expected non-empty gate reason")
	}
}

// bootstrap_pending must NOT be gated — gating it creates the Stage 0B
// chicken-and-egg deadlock (iSCSI removed → no writes → shipper never
// connects → mode never advances).
func TestT4_BootstrapPending_NotGated(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-bootstrap")

	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeBootstrapPending, Reason: "awaiting_shipper_connected"},
	}
	bs.coreProjMu.Unlock()

	bs.evaluateActivationGate(path)

	if gated, reason := bs.IsActivationGated(path); gated {
		t.Fatalf("bootstrap_pending must NOT be gated, got reason=%q", reason)
	}
}

// allocated_only must NOT be gated — fresh volume with no replicas yet.
func TestT4_AllocatedOnly_NotGated(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-allocated")

	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeAllocatedOnly},
	}
	bs.coreProjMu.Unlock()

	bs.evaluateActivationGate(path)

	if gated, reason := bs.IsActivationGated(path); gated {
		t.Fatalf("allocated_only must NOT be gated, got reason=%q", reason)
	}
}

// Transition from gated (degraded) to bootstrap_pending must clear the gate.
func TestT4_DegradedToBootstrapPending_ClearsGate(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-degrade-bootstrap")

	// Start gated.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "barrier_timeout"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, _ := bs.IsActivationGated(path); !gated {
		t.Fatal("expected gated for degraded")
	}

	// Transition to bootstrap_pending (recovery started).
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeBootstrapPending, Reason: "recovery_in_progress"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)
	if gated, reason := bs.IsActivationGated(path); gated {
		t.Fatalf("bootstrap_pending should clear gate, got reason=%q", reason)
	}
}

// P20-T4-C3: Missing projection with active V2 core fails closed.
func TestT4_MissingProjection_FailsClosed(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-missing-proj")

	// v2Core is non-nil (production config), but no projection cached.
	// This must fail closed, not silently leave serving enabled.
	if bs.v2Core == nil {
		t.Fatal("test requires v2Core != nil")
	}

	bs.evaluateActivationGate(path)

	gated, reason := bs.IsActivationGated(path)
	if !gated {
		t.Fatal("expected activation gated when V2 core is active but projection missing")
	}
	if reason != "missing_engine_projection" {
		t.Fatalf("reason=%q, want %q", reason, "missing_engine_projection")
	}
}

// P20-T4-C6: Gate actually removes iSCSI target (enforcement, not bookkeeping).
func TestT4_GateRemovesISCSITarget(t *testing.T) {
	bs := newTestBlockServiceDirect(t)
	path := createTestVolDirect(t, bs, "gate-iscsi-remove")

	// Create a real TargetServer (no listen, just registry).
	logger := log.New(os.Stderr, "iscsi-test: ", log.LstdFlags)
	ts := iscsi.NewTargetServer("127.0.0.1:0", iscsi.DefaultTargetConfig(), logger)
	bs.targetServer = ts

	// Register volume with target.
	vol, ok := bs.blockStore.GetBlockVolume(path)
	if !ok {
		t.Fatal("volume not found")
	}
	name := volumeNameFromPath(path)
	iqn := bs.iqnPrefix + blockvol.SanitizeIQN(name)
	adapter := blockvol.NewBlockVolAdapter(vol)
	ts.AddVolume(iqn, adapter)

	if !ts.HasTarget(iqn) {
		t.Fatal("target should exist before gate")
	}

	// Inject degraded projection and gate.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "test"},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)

	if ts.HasTarget(iqn) {
		t.Fatal("target should be removed after gate (enforcement, not just bookkeeping)")
	}

	// Inject healthy projection and ungate.
	bs.coreProjMu.Lock()
	bs.coreProj[path] = engine.PublicationProjection{
		Mode: engine.ModeView{Name: engine.ModePublishHealthy},
	}
	bs.coreProjMu.Unlock()
	bs.evaluateActivationGate(path)

	if !ts.HasTarget(iqn) {
		t.Fatal("target should be restored after ungate")
	}
}