seaweedfs/weed/server/qa_block_cp13_8a_test.go

package weed_server

import (
	"bytes"
	"context"
"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

// TestCP13_8A_ReplicaReadAfterReplication verifies that data replicated to
// a replica volume can be read back through the same BlockVol instance that
// the iSCSI adapter uses. This is the core CP13-8A investigation test.
//
// If this passes: the engine + weed integration is correct, and the CP13-8
// scenario failure is in the testrunner/cluster layer.
// If this fails: the bug is in the VS integration (adapter wiring, flusher, etc).
func TestCP13_8A_ReplicaReadAfterReplication(t *testing.T) {
	s := newSoakSetup(t)
	ctx := context.Background()

	// Create a volume (RF=2 sync_all).
	resp, err := s.ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name:           "cp13-8a-read",
		SizeBytes:      1 << 20,
		DurabilityMode: "sync_all",
	})
	if err != nil {
		t.Fatal(err)
	}
	primaryVS := resp.VolumeServer
	entry, _ := s.ms.blockRegistry.Lookup("cp13-8a-read")

	// Deliver assignments.
	s.bs.localServerID = primaryVS
	s.deliver(primaryVS)
	for _, ri := range entry.Replicas {
		s.deliver(ri.Server)
	}
	time.Sleep(200 * time.Millisecond)

	// Write data through the primary's BlockVol (simulating iSCSI write).
	primaryPath := entry.Path
	var primaryVol *blockvol.BlockVol
	s.store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) {
		if path == primaryPath {
			primaryVol = vol
		}
	})
	if primaryVol == nil {
		t.Fatal("primary volume not found in store")
	}

	// Write test pattern.
	testData := make([]byte, 4096)
	for i := range testData {
		testData[i] = byte(i % 251) // distinctive non-zero pattern
	}
	if err := primaryVol.WriteLBA(0, testData); err != nil {
		t.Fatalf("WriteLBA on primary: %v", err)
	}
	if err := primaryVol.WriteLBA(1, testData); err != nil {
		t.Fatalf("WriteLBA on primary LBA 1: %v", err)
	}

	// SyncCache to trigger sync_all barrier (replicates + confirms durability).
	if err := primaryVol.SyncCache(); err != nil {
		t.Fatalf("SyncCache: %v", err)
	}

	// Now find the REPLICA volume and read from it.
	// In the soak setup, both primary and replica are in the same store.
	var replicaPath string
	for _, ri := range entry.Replicas {
		replicaPath = ri.Path
		break
	}
	if replicaPath == "" {
		t.Fatal("no replica path found")
	}

	var replicaVol *blockvol.BlockVol
	s.store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) {
		if path == replicaPath {
			replicaVol = vol
		}
	})
	if replicaVol == nil {
		t.Fatal("replica volume not found in store")
	}

	// Let flusher run on replica (same as production).
	time.Sleep(200 * time.Millisecond)

	// Read from replica — this is what the iSCSI adapter would do.
	got, err := replicaVol.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA on replica: %v", err)
	}

	if bytes.Equal(got, make([]byte, 4096)) {
		t.Fatal("CP13-8A: replica ReadLBA returned all zeros — data not visible")
	}

	if !bytes.Equal(got, testData) {
		t.Fatalf("CP13-8A: replica data mismatch at LBA 0: got[0]=%d want[0]=%d", got[0], testData[0])
	}

	// Read LBA 1 too.
	got1, err := replicaVol.ReadLBA(1, 4096)
	if err != nil {
		t.Fatalf("ReadLBA on replica LBA 1: %v", err)
	}
	if !bytes.Equal(got1, testData) {
		t.Fatalf("CP13-8A: replica data mismatch at LBA 1")
	}

	// Also test through the adapter path (what iSCSI actually calls).
	adapter := blockvol.NewBlockVolAdapter(replicaVol)
	adapterGot, err := adapter.ReadAt(0, 4096)
	if err != nil {
		t.Fatalf("adapter ReadAt: %v", err)
	}
	if !bytes.Equal(adapterGot, testData) {
		t.Fatalf("CP13-8A: adapter data mismatch at LBA 0")
	}

	t.Log("CP13-8A: replica reads return correct data through both ReadLBA and adapter.ReadAt")
}

// TestCP13_8A_ReplicaReadAfterPromotion verifies that after promoting a
// replica to primary, the data is still readable.
func TestCP13_8A_ReplicaReadAfterPromotion(t *testing.T) {
	s := newSoakSetup(t)
	ctx := context.Background()

	resp, err := s.ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
		Name:           "cp13-8a-promote",
		SizeBytes:      1 << 20,
		DurabilityMode: "sync_all",
	})
	if err != nil {
		t.Fatal(err)
	}
	primaryVS := resp.VolumeServer
	entry, _ := s.ms.blockRegistry.Lookup("cp13-8a-promote")

	s.bs.localServerID = primaryVS
	s.deliver(primaryVS)
	for _, ri := range entry.Replicas {
		s.deliver(ri.Server)
	}
	time.Sleep(200 * time.Millisecond)

	// Write through primary.
	var primaryVol *blockvol.BlockVol
	s.store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) {
		if path == entry.Path {
			primaryVol = vol
		}
	})
	testData := make([]byte, 4096)
	for i := range testData {
		testData[i] = byte((i + 37) % 251)
	}
	if err := primaryVol.WriteLBA(0, testData); err != nil {
		t.Fatal(err)
	}
	if err := primaryVol.SyncCache(); err != nil {
		t.Fatal(err)
	}

	// Find replica.
	var replicaPath string
	for _, ri := range entry.Replicas {
		replicaPath = ri.Path
	}
	var replicaVol *blockvol.BlockVol
	s.store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) {
		if path == replicaPath {
			replicaVol = vol
		}
	})

	// Promote replica to primary.
	if err := replicaVol.HandleAssignment(2, blockvol.RolePrimary, 30*time.Second); err != nil {
		t.Fatalf("promote replica: %v", err)
	}

	// Read after promotion (let flusher run).
	time.Sleep(200 * time.Millisecond)
	got, err := replicaVol.ReadLBA(0, 4096)
	if err != nil {
		t.Fatalf("ReadLBA after promotion: %v", err)
	}
	if bytes.Equal(got, make([]byte, 4096)) {
		t.Fatal("CP13-8A: promoted replica reads zeros after promotion")
	}
	if !bytes.Equal(got, testData) {
		t.Fatalf("CP13-8A: promoted replica data mismatch")
	}

	t.Log("CP13-8A: promoted replica reads correct data")
}