refactoring

test/kafka/api_sequence_test.go

@@ -13,7 +13,7 @@ import (
 // TestKafkaGateway_APISequence logs all API requests that kafka-go makes
 func TestKafkaGateway_APISequence(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/client_integration_test.go

@@ -15,7 +15,7 @@ import (
 // TestKafkaGoClient_BasicProduceConsume tests our gateway with real kafka-go client
 func TestKafkaGoClient_BasicProduceConsume(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // Use random port
 	})
 
@@ -81,7 +81,7 @@ func TestKafkaGoClient_BasicProduceConsume(t *testing.T) {
 // TestKafkaGoClient_ConsumerGroups tests consumer group functionality
 func TestKafkaGoClient_ConsumerGroups(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 
@@ -134,7 +134,7 @@ func TestKafkaGoClient_MultiplePartitions(t *testing.T) {
 // TestKafkaGoClient_OffsetManagement tests offset commit/fetch operations
 func TestKafkaGoClient_OffsetManagement(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/comprehensive_e2e_test.go

@@ -15,7 +15,7 @@ import (
 // TestComprehensiveE2E tests both kafka-go and Sarama clients in a comprehensive scenario
 func TestComprehensiveE2E(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 
@@ -357,7 +357,7 @@ func testSaramaToKafkaGo(t *testing.T, addr, topic string) {
 // TestOffsetManagement tests offset commit and fetch operations
 func TestOffsetManagement(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/connection_close_debug_test.go

@@ -14,7 +14,7 @@ import (
 // TestConnectionCloseDebug captures the exact moment kafka-go closes the connection
 func TestConnectionCloseDebug(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 

test/kafka/connection_debug_test.go

@@ -13,7 +13,7 @@ import (
 // TestConnectionDebug debugs the exact connection behavior between kafka-go and our gateway
 func TestConnectionDebug(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 

test/kafka/consumer_group_debug_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestConsumerGroup_Debug(t *testing.T) {
 	// Start Kafka gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: ":0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 
 	go func() {
 		if err := gatewayServer.Start(); err != nil {

test/kafka/consumer_group_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestConsumerGroup_BasicFunctionality(t *testing.T) {
 	// Start Kafka gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: ":0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
@@ -203,7 +203,7 @@ func (h *ConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession,
 
 func TestConsumerGroup_OffsetCommitAndFetch(t *testing.T) {
 	// Start Kafka gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: ":0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
@@ -402,7 +402,7 @@ func (h *OffsetTestHandler) ConsumeClaim(session sarama.ConsumerGroupSession, cl
 
 func TestConsumerGroup_Rebalancing(t *testing.T) {
 	// Start Kafka gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: ":0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 
 	go func() {
 		if err := gatewayServer.Start(); err != nil {

test/kafka/consumer_only_test.go

@@ -13,7 +13,7 @@ import (
 // TestConsumerOnly tests only the consumer workflow to debug SyncGroup issue
 func TestConsumerOnly(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 

test/kafka/consumer_test.go

@@ -13,7 +13,7 @@ import (
 // TestKafkaGoReader tests if kafka-go Reader has different validation than Writer
 func TestKafkaGoReader(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/debug_connection_test.go

@@ -12,7 +12,7 @@ import (
 // TestGateway_BasicConnection tests if the gateway can handle basic TCP connections
 func TestGateway_BasicConnection(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 
@@ -37,7 +37,7 @@ func TestGateway_BasicConnection(t *testing.T) {
 // TestGateway_ApiVersionsRequest tests if we can send an ApiVersions request
 func TestGateway_ApiVersionsRequest(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 
@@ -138,7 +138,7 @@ func TestGateway_ApiVersionsRequest(t *testing.T) {
 // TestGateway_CreateTopicsRequest tests if we can send a CreateTopics request
 func TestGateway_CreateTopicsRequest(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/debug_consumer_group_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestDebugConsumerGroupWorkflow(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/debug_produce_response_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestDebugProduceV7Response(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/debug_produce_v7_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestDebugProduceV7Format(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/debug_readpartitions_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestDebugReadPartitions(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/e2e_test.go

@@ -12,7 +12,7 @@ import (
 // TestKafkaGateway_E2E tests the complete Kafka workflow using the gateway
 func TestKafkaGateway_E2E(t *testing.T) {
 	// Start the gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // use random port
 	})
 	
@@ -331,7 +331,7 @@ func TestKafkaGateway_ProduceConsume(t *testing.T) {
 // TestKafkaGateway_MultipleClients tests concurrent client connections
 func TestKafkaGateway_MultipleClients(t *testing.T) {
 	// Start the gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // use random port
 	})
 	
@@ -393,7 +393,7 @@ func TestKafkaGateway_StressTest(t *testing.T) {
 	}
 	
 	// Start the gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 	

test/kafka/gateway_smoke_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestGateway_StartAcceptsConnections(t *testing.T) {
-	srv := gateway.NewServer(gateway.Options{Listen: ":0"})
+	srv := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 	if err := srv.Start(); err != nil {
 		t.Fatalf("start gateway: %v", err)
 	}
@@ -28,7 +28,7 @@ func TestGateway_StartAcceptsConnections(t *testing.T) {
 }
 
 func TestGateway_RefusesAfterClose(t *testing.T) {
-	srv := gateway.NewServer(gateway.Options{Listen: ":0"})
+	srv := gateway.NewTestServer(gateway.Options{Listen: ":0"})
 	if err := srv.Start(); err != nil {
 		t.Fatalf("start gateway: %v", err)
 	}

test/kafka/joingroup_debug_test.go

@@ -13,7 +13,7 @@ import (
 // TestJoinGroupDebug captures the exact JoinGroup request/response to debug format issues
 func TestJoinGroupDebug(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 

test/kafka/kafka_go_debug_test.go

@@ -14,7 +14,7 @@ import (
 // TestKafkaGoDeepDebug attempts to get more detailed error information from kafka-go
 func TestKafkaGoDeepDebug(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/kafka_go_internal_debug_test.go

@@ -15,7 +15,7 @@ import (
 // TestKafkaGoInternalDebug attempts to debug kafka-go's internal parsing by intercepting the read operations
 func TestKafkaGoInternalDebug(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/kafka_go_metadata_test.go

@@ -12,7 +12,7 @@ import (
 // TestKafkaGoMetadataV1Compatibility tests if our Metadata v1 response is compatible with kafka-go
 func TestKafkaGoMetadataV1Compatibility(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 

test/kafka/kafka_go_produce_only_test.go

@@ -11,8 +11,10 @@ import (
 )
 
 func TestKafkaGo_ProduceOnly(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
+	// Start gateway with test server
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen: "127.0.0.1:0",
+	})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)
@@ -47,7 +49,7 @@ func TestKafkaGo_ProduceOnly(t *testing.T) {
 
 func TestKafkaGo_ProduceConsume(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: "127.0.0.1:0"})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)
@@ -112,7 +114,7 @@ func TestKafkaGo_ProduceConsume(t *testing.T) {
 			t.Fatalf("kafka-go consume failed at message %d: %v", i, err)
 		}
 		consumedMessages = append(consumedMessages, msg)
-		t.Logf("✅ Consumed message %d: key=%s, value=%s, offset=%d", 
+		t.Logf("✅ Consumed message %d: key=%s, value=%s, offset=%d",
 			i, string(msg.Key), string(msg.Value), msg.Offset)
 	}
 
@@ -124,11 +126,11 @@ func TestKafkaGo_ProduceConsume(t *testing.T) {
 	for i, consumed := range consumedMessages {
 		expected := testMessages[i]
 		if string(consumed.Key) != string(expected.Key) {
-			t.Errorf("Message %d key mismatch: got %s, want %s", 
+			t.Errorf("Message %d key mismatch: got %s, want %s",
 				i, string(consumed.Key), string(expected.Key))
 		}
 		if string(consumed.Value) != string(expected.Value) {
-			t.Errorf("Message %d value mismatch: got %s, want %s", 
+			t.Errorf("Message %d value mismatch: got %s, want %s",
 				i, string(consumed.Value), string(expected.Value))
 		}
 	}
@@ -138,7 +140,7 @@ func TestKafkaGo_ProduceConsume(t *testing.T) {
 
 func TestKafkaGo_ConsumerGroup(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: "127.0.0.1:0"})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)
@@ -205,7 +207,7 @@ func TestKafkaGo_ConsumerGroup(t *testing.T) {
 			t.Fatalf("kafka-go consumer group failed at message %d: %v", i, err)
 		}
 		consumedMessages = append(consumedMessages, msg)
-		t.Logf("✅ Consumer group consumed message %d: key=%s, value=%s, offset=%d", 
+		t.Logf("✅ Consumer group consumed message %d: key=%s, value=%s, offset=%d",
 			i, string(msg.Key), string(msg.Value), msg.Offset)
 	}
 
@@ -217,11 +219,11 @@ func TestKafkaGo_ConsumerGroup(t *testing.T) {
 	for i, consumed := range consumedMessages {
 		expected := testMessages[i]
 		if string(consumed.Key) != string(expected.Key) {
-			t.Errorf("Message %d key mismatch: got %s, want %s", 
+			t.Errorf("Message %d key mismatch: got %s, want %s",
 				i, string(consumed.Key), string(expected.Key))
 		}
 		if string(consumed.Value) != string(expected.Value) {
-			t.Errorf("Message %d value mismatch: got %s, want %s", 
+			t.Errorf("Message %d value mismatch: got %s, want %s",
 				i, string(consumed.Value), string(expected.Value))
 		}
 	}

test/kafka/metadata_comparison_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestMetadataResponseComparison(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: "127.0.0.1:0"})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)

test/kafka/metadata_debug_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestMetadataV6Debug(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
+	gatewayServer := gateway.NewTestServer(gateway.Options{Listen: "127.0.0.1:0"})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)

test/kafka/metadata_v1_isolation_test.go

@@ -15,7 +15,7 @@ import (
 // TestMetadataV1Isolation creates a minimal test to isolate the Metadata v1 parsing issue
 func TestMetadataV1Isolation(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/mock_smq_handler.go

@@ -0,0 +1,293 @@
+package kafka
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/offset"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// MockSMQHandler provides a realistic SeaweedMQ simulation for testing
+// It behaves like the real SeaweedMQ integration but uses in-memory storage
+type MockSMQHandler struct {
+	// Topic management
+	topicsMu sync.RWMutex
+	topics   map[string]*MockTopicInfo
+
+	// Message storage - simulates SeaweedMQ's persistent storage
+	messagesMu sync.RWMutex
+	messages   map[string]map[int32][]*MockSMQRecord // topic -> partition -> []records
+
+	// Offset management - simulates Kafka offset ledgers
+	ledgersMu sync.RWMutex
+	ledgers   map[string]*offset.Ledger // topic-partition -> ledger
+
+	// Simulated SMQ timestamp tracking
+	lastTimestamp map[string]map[int32]int64 // topic -> partition -> last_timestamp
+}
+
+// MockTopicInfo represents a Kafka topic in the mock SMQ environment
+type MockTopicInfo struct {
+	Name       string
+	Partitions int32
+	CreatedAt  int64
+	Schema     *schema_pb.Topic
+}
+
+// MockSMQRecord represents a record in the mock SeaweedMQ storage
+type MockSMQRecord struct {
+	Key       []byte
+	Value     []byte
+	Timestamp int64 // SeaweedMQ timestamp (nanoseconds)
+	Offset    int64 // Kafka offset for this partition
+}
+
+// Implement SMQRecord interface
+func (r *MockSMQRecord) GetKey() []byte      { return r.Key }
+func (r *MockSMQRecord) GetValue() []byte    { return r.Value }
+func (r *MockSMQRecord) GetTimestamp() int64 { return r.Timestamp }
+func (r *MockSMQRecord) GetOffset() int64    { return r.Offset }
+
+// NewMockSMQHandler creates a new mock SeaweedMQ handler for testing
+func NewMockSMQHandler() *MockSMQHandler {
+	return &MockSMQHandler{
+		topics:        make(map[string]*MockTopicInfo),
+		messages:      make(map[string]map[int32][]*MockSMQRecord),
+		ledgers:       make(map[string]*offset.Ledger),
+		lastTimestamp: make(map[string]map[int32]int64),
+	}
+}
+
+// TopicExists checks if a topic exists in the mock SMQ
+func (m *MockSMQHandler) TopicExists(topic string) bool {
+	m.topicsMu.RLock()
+	defer m.topicsMu.RUnlock()
+	_, exists := m.topics[topic]
+	return exists
+}
+
+// ListTopics returns all topics in the mock SMQ
+func (m *MockSMQHandler) ListTopics() []string {
+	m.topicsMu.RLock()
+	defer m.topicsMu.RUnlock()
+
+	topics := make([]string, 0, len(m.topics))
+	for name := range m.topics {
+		topics = append(topics, name)
+	}
+	return topics
+}
+
+// CreateTopic creates a new topic in the mock SMQ
+func (m *MockSMQHandler) CreateTopic(topic string, partitions int32) error {
+	m.topicsMu.Lock()
+	defer m.topicsMu.Unlock()
+
+	if _, exists := m.topics[topic]; exists {
+		return fmt.Errorf("topic %s already exists", topic)
+	}
+
+	// Create topic info
+	m.topics[topic] = &MockTopicInfo{
+		Name:       topic,
+		Partitions: partitions,
+		CreatedAt:  time.Now().UnixNano(),
+		Schema: &schema_pb.Topic{
+			Name: topic,
+			// PartitionCount removed - not part of schema_pb.Topic
+		},
+	}
+
+	// Initialize message storage for all partitions
+	m.messagesMu.Lock()
+	m.messages[topic] = make(map[int32][]*MockSMQRecord)
+	for i := int32(0); i < partitions; i++ {
+		m.messages[topic][i] = make([]*MockSMQRecord, 0, 1000)
+	}
+	m.messagesMu.Unlock()
+
+	// Initialize timestamp tracking
+	if m.lastTimestamp[topic] == nil {
+		m.lastTimestamp[topic] = make(map[int32]int64)
+	}
+	for i := int32(0); i < partitions; i++ {
+		m.lastTimestamp[topic][i] = 0
+	}
+
+	fmt.Printf("MOCK SMQ: Created topic %s with %d partitions\n", topic, partitions)
+	return nil
+}
+
+// DeleteTopic removes a topic from the mock SMQ
+func (m *MockSMQHandler) DeleteTopic(topic string) error {
+	m.topicsMu.Lock()
+	defer m.topicsMu.Unlock()
+
+	if _, exists := m.topics[topic]; !exists {
+		return fmt.Errorf("topic %s does not exist", topic)
+	}
+
+	delete(m.topics, topic)
+
+	// Clean up messages
+	m.messagesMu.Lock()
+	delete(m.messages, topic)
+	m.messagesMu.Unlock()
+
+	// Clean up ledgers
+	m.ledgersMu.Lock()
+	keysToDelete := make([]string, 0)
+	for key := range m.ledgers {
+		if key[:len(topic)+1] == topic+"-" {
+			keysToDelete = append(keysToDelete, key)
+		}
+	}
+	for _, key := range keysToDelete {
+		delete(m.ledgers, key)
+	}
+	m.ledgersMu.Unlock()
+
+	fmt.Printf("MOCK SMQ: Deleted topic %s\n", topic)
+	return nil
+}
+
+// GetOrCreateLedger gets or creates a Kafka offset ledger for a topic-partition
+func (m *MockSMQHandler) GetOrCreateLedger(topic string, partition int32) *offset.Ledger {
+	key := fmt.Sprintf("%s-%d", topic, partition)
+
+	m.ledgersMu.Lock()
+	defer m.ledgersMu.Unlock()
+
+	if ledger, exists := m.ledgers[key]; exists {
+		return ledger
+	}
+
+	// Create new ledger
+	ledger := offset.NewLedger()
+	m.ledgers[key] = ledger
+
+	// Ensure topic is created
+	if !m.TopicExists(topic) {
+		m.CreateTopic(topic, partition+1) // Ensure enough partitions
+	}
+
+	return ledger
+}
+
+// GetLedger retrieves an existing ledger or returns nil
+func (m *MockSMQHandler) GetLedger(topic string, partition int32) *offset.Ledger {
+	key := fmt.Sprintf("%s-%d", topic, partition)
+
+	m.ledgersMu.RLock()
+	defer m.ledgersMu.RUnlock()
+
+	return m.ledgers[key]
+}
+
+// ProduceRecord publishes a record to the mock SeaweedMQ and updates Kafka offset tracking
+func (m *MockSMQHandler) ProduceRecord(topic string, partition int32, key, value []byte) (int64, error) {
+	// Verify topic exists
+	if !m.TopicExists(topic) {
+		return 0, fmt.Errorf("topic %s does not exist", topic)
+	}
+
+	// Get current timestamp (simulate SeaweedMQ timestamp)
+	timestamp := time.Now().UnixNano()
+
+	// Get or create Kafka offset ledger
+	ledger := m.GetOrCreateLedger(topic, partition)
+
+	// Assign Kafka offset
+	kafkaOffset := ledger.AssignOffsets(1)
+
+	// Record in ledger (simulates Kafka offset -> SMQ timestamp mapping)
+	messageSize := int32(len(value))
+	if err := ledger.AppendRecord(kafkaOffset, timestamp, messageSize); err != nil {
+		return 0, fmt.Errorf("failed to append to ledger: %w", err)
+	}
+
+	// Store message in mock SMQ storage (simulates persistent storage)
+	m.messagesMu.Lock()
+	defer m.messagesMu.Unlock()
+
+	if m.messages[topic] == nil {
+		m.messages[topic] = make(map[int32][]*MockSMQRecord)
+	}
+	if m.messages[topic][partition] == nil {
+		m.messages[topic][partition] = make([]*MockSMQRecord, 0, 1000)
+	}
+
+	// Create message record (simulate SMQ storage)
+	record := &MockSMQRecord{
+		Key:       append([]byte(nil), key...),   // Deep copy key
+		Value:     append([]byte(nil), value...), // Deep copy value
+		Timestamp: timestamp,
+		Offset:    kafkaOffset,
+	}
+
+	// Append to partition (simulate SMQ append-only log)
+	m.messages[topic][partition] = append(m.messages[topic][partition], record)
+
+	// Update last timestamp
+	m.lastTimestamp[topic][partition] = timestamp
+
+	fmt.Printf("MOCK SMQ: Stored record - topic:%s, partition:%d, kafka_offset:%d, smq_timestamp:%d, key:%s, value:%s\n",
+		topic, partition, kafkaOffset, timestamp, string(key), string(value))
+
+	return kafkaOffset, nil
+}
+
+// GetStoredRecords retrieves records from mock SMQ storage starting from a given Kafka offset
+func (m *MockSMQHandler) GetStoredRecords(topic string, partition int32, fromKafkaOffset int64, maxRecords int) ([]offset.SMQRecord, error) {
+	m.messagesMu.RLock()
+	defer m.messagesMu.RUnlock()
+
+	if m.messages[topic] == nil || m.messages[topic][partition] == nil {
+		return nil, nil // No messages
+	}
+
+	records := m.messages[topic][partition]
+	result := make([]offset.SMQRecord, 0, maxRecords)
+
+	// Find records starting from the given Kafka offset
+	for _, record := range records {
+		if record.Offset >= fromKafkaOffset && len(result) < maxRecords {
+			result = append(result, record)
+		}
+	}
+
+	fmt.Printf("MOCK SMQ: Retrieved %d records from topic:%s, partition:%d, from_offset:%d\n",
+		len(result), topic, partition, fromKafkaOffset)
+
+	return result, nil
+}
+
+// Close shuts down the mock SMQ handler
+func (m *MockSMQHandler) Close() error {
+	fmt.Printf("MOCK SMQ: Handler closed\n")
+	return nil
+}
+
+// GetPartitionCount returns the number of partitions for a topic
+func (m *MockSMQHandler) GetPartitionCount(topic string) int32 {
+	m.topicsMu.RLock()
+	defer m.topicsMu.RUnlock()
+
+	if topicInfo, exists := m.topics[topic]; exists {
+		return topicInfo.Partitions
+	}
+	return 0
+}
+
+// GetMessageCount returns the total number of messages stored for a topic-partition
+func (m *MockSMQHandler) GetMessageCount(topic string, partition int32) int {
+	m.messagesMu.RLock()
+	defer m.messagesMu.RUnlock()
+
+	if m.messages[topic] == nil || m.messages[topic][partition] == nil {
+		return 0
+	}
+	return len(m.messages[topic][partition])
+}

test/kafka/network_capture_test.go

@@ -14,7 +14,7 @@ import (
 // TestNetworkCapture captures the exact bytes sent over the network
 func TestNetworkCapture(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/parsing_debug_test.go

@@ -16,7 +16,7 @@ import (
 // TestParsingDebug attempts to manually replicate kafka-go's parsing logic
 func TestParsingDebug(t *testing.T) {
 	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/produce_consume_cycle_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestKafkaProduceConsumeE2E(t *testing.T) {
 	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 	

test/kafka/produce_consume_test.go

@@ -14,7 +14,7 @@ import (
 // TestKafkaGoClient_DirectProduceConsume bypasses CreateTopics and tests produce/consume directly
 func TestKafkaGoClient_DirectProduceConsume(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/raw_protocol_test.go

@@ -12,7 +12,7 @@ import (
 // TestRawProduceRequest tests our Produce API directly without kafka-go
 func TestRawProduceRequest(t *testing.T) {
 	// Start the gateway server
-	srv := gateway.NewServer(gateway.Options{
+	srv := gateway.NewTestServer(gateway.Options{
 		Listen: ":0",
 	})
 

test/kafka/sarama_e2e_test.go

@@ -11,11 +11,14 @@ import (
 )
 
 func TestSaramaE2EProduceConsume(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	// Start gateway with test server (creates in-memory test handler)
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 
+	// Note: NewTestServer creates an in-memory handler for testing only
+	// Production deployments use NewServer() and require real SeaweedMQ masters
+
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
 			t.Errorf("Failed to start gateway: %v", err)
@@ -115,8 +118,8 @@ func TestSaramaE2EProduceConsume(t *testing.T) {
 }
 
 func TestSaramaConsumerGroup(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
+	// Start gateway with test server
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: "127.0.0.1:0",
 	})
 

test/kafka/sarama_simple_test.go

@@ -10,9 +10,10 @@ import (
 )
 
 func TestSaramaSimpleProducer(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: "127.0.0.1:0",
+	// Start gateway with test mode
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen:  "127.0.0.1:0",
+		
 	})
 
 	go func() {
@@ -85,9 +86,10 @@ func TestSaramaSimpleProducer(t *testing.T) {
 }
 
 func TestSaramaMinimalConfig(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: "127.0.0.1:0",
+	// Start gateway with test mode
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen:  "127.0.0.1:0",
+		
 	})
 
 	go func() {
@@ -147,9 +149,10 @@ func TestSaramaMinimalConfig(t *testing.T) {
 }
 
 func TestSaramaProduceConsume(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: "127.0.0.1:0",
+	// Start gateway with test mode
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen:  "127.0.0.1:0",
+		
 	})
 
 	go func() {

test/kafka/sarama_test.go

@@ -12,9 +12,10 @@ import (
 
 // TestSaramaCompatibility tests our Kafka gateway with IBM Sarama client
 func TestSaramaCompatibility(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: "127.0.0.1:0",
+	// Start gateway with test mode
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen:  "127.0.0.1:0",
+		
 	})
 
 	go gatewayServer.Start()
@@ -263,9 +264,10 @@ func (h *SaramaConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSe
 
 // TestSaramaMetadataOnly tests just the metadata functionality that's failing with kafka-go
 func TestSaramaMetadataOnly(t *testing.T) {
-	// Start gateway
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: "127.0.0.1:0",
+	// Start gateway with test mode
+	gatewayServer := gateway.NewTestServer(gateway.Options{
+		Listen:  "127.0.0.1:0",
+		
 	})
 
 	go gatewayServer.Start()

test/kafka/seaweedmq_integration_test.go

@@ -15,7 +15,7 @@ func TestSeaweedMQIntegration_E2E(t *testing.T) {
 	t.Skip("Integration test requires real SeaweedMQ setup - run manually")
 
 	// Start the gateway with SeaweedMQ backend
-	gatewayServer := gateway.NewServer(gateway.Options{
+	gatewayServer := gateway.NewTestServer(gateway.Options{
 		Listen: ":0", // random port
 	})
 
@@ -235,7 +235,7 @@ func buildCreateTopicsRequestCustom(topicName string) []byte {
 func TestSeaweedMQGateway_ModeSelection(t *testing.T) {
 	// Test in-memory mode (should always work)
 	t.Run("InMemoryMode", func(t *testing.T) {
-		server := gateway.NewServer(gateway.Options{
+		server := gateway.NewTestServer(gateway.Options{
 			Listen: ":0",
 		})
 
@@ -255,7 +255,7 @@ func TestSeaweedMQGateway_ModeSelection(t *testing.T) {
 
 	// Test SeaweedMQ mode with invalid agent (should fall back)
 	t.Run("SeaweedMQModeFallback", func(t *testing.T) {
-		server := gateway.NewServer(gateway.Options{
+		server := gateway.NewTestServer(gateway.Options{
 			Listen: ":0",
 		})
 
@@ -306,7 +306,7 @@ func TestSeaweedMQGateway_ConfigValidation(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			server := gateway.NewServer(tc.options)
+			server := gateway.NewTestServer(tc.options)
 			err := server.Start()
 
 			if tc.shouldWork && err != nil {