log read stateless

weed/util/log_buffer/log_read_integration_test.go

@@ -0,0 +1,341 @@
+package log_buffer
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// TestConcurrentProducerConsumer simulates the integration test scenario:
+// - One producer writing messages continuously
+// - Multiple consumers reading from different offsets
+// - Consumers reading sequentially (like Kafka consumers)
+func TestConcurrentProducerConsumer(t *testing.T) {
+	lb := NewLogBuffer("integration-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numConsumers = 2
+	const messagesPerConsumer = numMessages / numConsumers
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			lb.AddLogEntryToBuffer(entry)
+			time.Sleep(1 * time.Millisecond) // Simulate production rate
+		}
+		producerDone <- true
+	}()
+
+	// Start consumers
+	consumerWg := sync.WaitGroup{}
+	consumerErrors := make(chan error, numConsumers)
+	consumedCounts := make([]int64, numConsumers)
+
+	for consumerID := 0; consumerID < numConsumers; consumerID++ {
+		consumerWg.Add(1)
+		go func(id int, startOffset int64) {
+			defer consumerWg.Done()
+
+			currentOffset := startOffset
+			for currentOffset < startOffset+int64(messagesPerConsumer) {
+				// Read 10 messages at a time (like integration test)
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					consumerErrors <- err
+					return
+				}
+
+				if len(messages) == 0 {
+					// No data yet, wait a bit
+					time.Sleep(5 * time.Millisecond)
+					continue
+				}
+
+				// Verify sequential offsets
+				for i, msg := range messages {
+					expectedOffset := currentOffset + int64(i)
+					if msg.Offset != expectedOffset {
+						t.Errorf("Consumer %d: Expected offset %d, got %d", id, expectedOffset, msg.Offset)
+					}
+				}
+
+				atomic.AddInt64(&consumedCounts[id], int64(len(messages)))
+				currentOffset = nextOffset
+			}
+		}(consumerID, int64(consumerID*messagesPerConsumer))
+	}
+
+	// Wait for producer to finish
+	<-producerDone
+
+	// Wait for consumers (with timeout)
+	done := make(chan bool)
+	go func() {
+		consumerWg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-consumerErrors:
+		t.Fatalf("Consumer error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for consumers to finish")
+	}
+
+	// Verify all messages were consumed
+	totalConsumed := int64(0)
+	for i, count := range consumedCounts {
+		t.Logf("Consumer %d consumed %d messages", i, count)
+		totalConsumed += count
+	}
+
+	if totalConsumed != numMessages {
+		t.Errorf("Expected to consume %d messages, but consumed %d", numMessages, totalConsumed)
+	}
+}
+
+// TestBackwardSeeksWhileProducing simulates consumer rebalancing where
+// consumers seek backward to earlier offsets while producer is still writing
+func TestBackwardSeeksWhileProducing(t *testing.T) {
+	lb := NewLogBuffer("backward-seek-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 500
+	const numSeeks = 10
+
+	// Start producer
+	producerDone := make(chan bool)
+	go func() {
+		for i := 0; i < numMessages; i++ {
+			entry := &filer_pb.LogEntry{
+				TsNs:   time.Now().UnixNano(),
+				Key:    []byte("key"),
+				Data:   []byte("value"),
+				Offset: int64(i),
+			}
+			lb.AddLogEntryToBuffer(entry)
+			time.Sleep(1 * time.Millisecond)
+		}
+		producerDone <- true
+	}()
+
+	// Consumer that seeks backward periodically
+	consumerDone := make(chan bool)
+	readOffsets := make(map[int64]int) // Track how many times each offset was read
+
+	go func() {
+		currentOffset := int64(0)
+		seeksRemaining := numSeeks
+
+		for currentOffset < numMessages {
+			// Read some messages
+			messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+			if err != nil {
+				t.Errorf("Read error: %v", err)
+				consumerDone <- true
+				return
+			}
+
+			if len(messages) == 0 {
+				time.Sleep(5 * time.Millisecond)
+				continue
+			}
+
+			// Track read offsets
+			for _, msg := range messages {
+				readOffsets[msg.Offset]++
+			}
+
+			// Periodically seek backward (simulating rebalancing)
+			if seeksRemaining > 0 && nextOffset > 50 && nextOffset%100 == 0 {
+				seekOffset := nextOffset - 20
+				t.Logf("Seeking backward from %d to %d", nextOffset, seekOffset)
+				currentOffset = seekOffset
+				seeksRemaining--
+			} else {
+				currentOffset = nextOffset
+			}
+		}
+
+		consumerDone <- true
+	}()
+
+	// Wait for both
+	<-producerDone
+	<-consumerDone
+
+	// Verify each offset was read at least once
+	for i := int64(0); i < numMessages; i++ {
+		if readOffsets[i] == 0 {
+			t.Errorf("Offset %d was never read", i)
+		}
+	}
+
+	t.Logf("Total unique offsets read: %d out of %d", len(readOffsets), numMessages)
+}
+
+// TestHighConcurrencyReads simulates multiple consumers reading from
+// different offsets simultaneously (stress test)
+func TestHighConcurrencyReads(t *testing.T) {
+	lb := NewLogBuffer("high-concurrency-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 1000
+	const numReaders = 10
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Start many concurrent readers at different offsets
+	wg := sync.WaitGroup{}
+	errors := make(chan error, numReaders)
+
+	for reader := 0; reader < numReaders; reader++ {
+		wg.Add(1)
+		go func(startOffset int64) {
+			defer wg.Done()
+
+			// Read 100 messages from this offset
+			currentOffset := startOffset
+			readCount := 0
+
+			for readCount < 100 && currentOffset < numMessages {
+				messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
+				if err != nil {
+					errors <- err
+					return
+				}
+
+				// Verify offsets are sequential
+				for i, msg := range messages {
+					expected := currentOffset + int64(i)
+					if msg.Offset != expected {
+						t.Errorf("Reader at %d: expected offset %d, got %d", startOffset, expected, msg.Offset)
+					}
+				}
+
+				readCount += len(messages)
+				currentOffset = nextOffset
+			}
+		}(int64(reader * 10))
+	}
+
+	// Wait with timeout
+	done := make(chan bool)
+	go func() {
+		wg.Wait()
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// Success
+	case err := <-errors:
+		t.Fatalf("Reader error: %v", err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Timeout waiting for readers")
+	}
+}
+
+// TestRepeatedReadsAtSameOffset simulates what happens when Kafka
+// consumer re-fetches the same offset multiple times (due to timeouts or retries)
+func TestRepeatedReadsAtSameOffset(t *testing.T) {
+	lb := NewLogBuffer("repeated-reads-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	const numMessages = 100
+
+	// Pre-populate buffer
+	for i := 0; i < numMessages; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Read the same offset multiple times concurrently
+	const numReads = 10
+	const testOffset = int64(50)
+
+	wg := sync.WaitGroup{}
+	results := make([][]*filer_pb.LogEntry, numReads)
+
+	for i := 0; i < numReads; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			messages, _, _, _, err := lb.ReadMessagesAtOffset(testOffset, 10, 10240)
+			if err != nil {
+				t.Errorf("Read %d error: %v", idx, err)
+				return
+			}
+			results[idx] = messages
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Verify all reads returned the same data
+	firstRead := results[0]
+	for i := 1; i < numReads; i++ {
+		if len(results[i]) != len(firstRead) {
+			t.Errorf("Read %d returned %d messages, expected %d", i, len(results[i]), len(firstRead))
+		}
+
+		for j := range results[i] {
+			if results[i][j].Offset != firstRead[j].Offset {
+				t.Errorf("Read %d message %d has offset %d, expected %d",
+					i, j, results[i][j].Offset, firstRead[j].Offset)
+			}
+		}
+	}
+}
+
+// TestEmptyPartitionPolling simulates consumers polling empty partitions
+// waiting for data (common in Kafka)
+func TestEmptyPartitionPolling(t *testing.T) {
+	lb := NewLogBuffer("empty-partition-test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Try to read from empty partition
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 0 {
+		t.Errorf("Expected nextOffset=0, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}

weed/util/log_buffer/log_read_stateless.go

@@ -0,0 +1,321 @@
+package log_buffer
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// ReadMessagesAtOffset provides Kafka-style stateless reads from LogBuffer
+// Each call is completely independent - no state maintained between calls
+// Thread-safe for concurrent reads at different offsets
+//
+// This is the recommended API for stateless clients like Kafka gateway
+// Unlike Subscribe loops, this:
+// 1. Returns immediately with available data (or empty if none)
+// 2. Does not maintain any session state
+// 3. Safe for concurrent calls
+// 4. No cancellation/restart complexity
+//
+// Returns:
+// - messages: Array of messages starting at startOffset
+// - nextOffset: Offset to use for next fetch
+// - highWaterMark: Highest offset available in partition
+// - endOfPartition: True if no more data available
+// - err: Any error encountered
+func (logBuffer *LogBuffer) ReadMessagesAtOffset(startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	highWaterMark int64,
+	endOfPartition bool,
+	err error,
+) {
+	glog.V(4).Infof("[StatelessRead] Reading from offset %d, maxMessages=%d, maxBytes=%d",
+		startOffset, maxMessages, maxBytes)
+
+	// Quick validation
+	if maxMessages <= 0 {
+		maxMessages = 100 // Default reasonable batch size
+	}
+	if maxBytes <= 0 {
+		maxBytes = 4 * 1024 * 1024 // 4MB default
+	}
+
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+
+	// Try to read from in-memory buffers first (hot path)
+	logBuffer.RLock()
+	currentBufferEnd := logBuffer.offset
+	bufferStartOffset := logBuffer.bufferStartOffset
+	highWaterMark = currentBufferEnd
+
+	// Check if requested offset is in current buffer
+	if startOffset >= bufferStartOffset && startOffset <= currentBufferEnd {
+		// Read from current buffer
+		glog.V(4).Infof("[StatelessRead] Reading from current buffer: start=%d, end=%d",
+			bufferStartOffset, currentBufferEnd)
+
+		if logBuffer.pos > 0 {
+			// Make a copy of the buffer to avoid concurrent modification
+			bufCopy := make([]byte, logBuffer.pos)
+			copy(bufCopy, logBuffer.buf[:logBuffer.pos])
+			logBuffer.RUnlock() // Release lock early
+
+			// Parse messages from buffer copy
+			messages, nextOffset, _, err = parseMessagesFromBuffer(
+				bufCopy, startOffset, maxMessages, maxBytes)
+
+			if err != nil {
+				return nil, startOffset, highWaterMark, false, err
+			}
+
+			glog.V(4).Infof("[StatelessRead] Read %d messages from current buffer, nextOffset=%d",
+				len(messages), nextOffset)
+
+			// Check if we reached the end
+			endOfPartition = (nextOffset >= currentBufferEnd) && (len(messages) == 0 || len(messages) < maxMessages)
+			return messages, nextOffset, highWaterMark, endOfPartition, nil
+		}
+
+		// Buffer is empty but offset is in range - check previous buffers
+		logBuffer.RUnlock()
+
+		// Try previous buffers
+		logBuffer.RLock()
+		for _, prevBuf := range logBuffer.prevBuffers.buffers {
+			if startOffset >= prevBuf.startOffset && startOffset <= prevBuf.offset {
+				if prevBuf.size > 0 {
+					// Found in previous buffer
+					bufCopy := make([]byte, prevBuf.size)
+					copy(bufCopy, prevBuf.buf[:prevBuf.size])
+					logBuffer.RUnlock()
+
+					messages, nextOffset, _, err = parseMessagesFromBuffer(
+						bufCopy, startOffset, maxMessages, maxBytes)
+
+					if err != nil {
+						return nil, startOffset, highWaterMark, false, err
+					}
+
+					glog.V(4).Infof("[StatelessRead] Read %d messages from previous buffer, nextOffset=%d",
+						len(messages), nextOffset)
+
+					endOfPartition = false // More data might be in current buffer
+					return messages, nextOffset, highWaterMark, endOfPartition, nil
+				}
+				// Empty previous buffer means data was flushed
+				break
+			}
+		}
+		logBuffer.RUnlock()
+
+		// Data not in memory - for stateless fetch, we don't do disk I/O to avoid blocking
+		// Return empty with offset out of range indication
+		glog.V(2).Infof("[StatelessRead] Data at offset %d not in memory (buffer: %d-%d), returning empty",
+			startOffset, bufferStartOffset, currentBufferEnd)
+		return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d out of range (in-memory: %d-%d)",
+			startOffset, bufferStartOffset, currentBufferEnd)
+	}
+
+	logBuffer.RUnlock()
+
+	// Offset is not in current buffer range
+	if startOffset < bufferStartOffset {
+		// Historical data - for stateless fetch, we don't do disk I/O to avoid blocking
+		// Return empty with offset out of range indication
+		glog.V(2).Infof("[StatelessRead] Requested offset %d < buffer start %d (too old), returning empty",
+			startOffset, bufferStartOffset)
+		return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d)",
+			startOffset, bufferStartOffset)
+	}
+
+	// startOffset > currentBufferEnd - future offset, no data available yet
+	glog.V(4).Infof("[StatelessRead] Future offset %d > buffer end %d, no data available",
+		startOffset, currentBufferEnd)
+	return messages, startOffset, highWaterMark, true, nil
+}
+
+// parseMessagesFromBuffer parses messages from a buffer byte slice
+// This is thread-safe as it operates on a copy of the buffer
+func parseMessagesFromBuffer(buf []byte, startOffset int64, maxMessages int, maxBytes int) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	totalBytes int,
+	err error,
+) {
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+	totalBytes = 0
+	foundStart := false
+
+	for pos := 0; pos+4 < len(buf) && len(messages) < maxMessages && totalBytes < maxBytes; {
+		// Read message size
+		size := util.BytesToUint32(buf[pos : pos+4])
+		if pos+4+int(size) > len(buf) {
+			// Incomplete message at end of buffer
+			glog.V(4).Infof("[parseMessages] Incomplete message at pos %d, size %d, bufLen %d",
+				pos, size, len(buf))
+			break
+		}
+
+		// Parse message
+		entryData := buf[pos+4 : pos+4+int(size)]
+		logEntry := &filer_pb.LogEntry{}
+		if err = proto.Unmarshal(entryData, logEntry); err != nil {
+			glog.Warningf("[parseMessages] Failed to unmarshal message: %v", err)
+			pos += 4 + int(size)
+			continue
+		}
+
+		// Initialize foundStart from first message
+		if !foundStart {
+			// Find the first message at or after startOffset
+			if logEntry.Offset >= startOffset {
+				foundStart = true
+				nextOffset = logEntry.Offset
+			} else {
+				// Skip messages before startOffset
+				pos += 4 + int(size)
+				continue
+			}
+		}
+
+		// Check if this message matches expected offset
+		if foundStart && logEntry.Offset >= startOffset {
+			messages = append(messages, logEntry)
+			totalBytes += 4 + int(size)
+			nextOffset = logEntry.Offset + 1
+		}
+
+		pos += 4 + int(size)
+	}
+
+	glog.V(4).Infof("[parseMessages] Parsed %d messages, nextOffset=%d, totalBytes=%d",
+		len(messages), nextOffset, totalBytes)
+
+	return messages, nextOffset, totalBytes, nil
+}
+
+// readMessagesFromDisk reads messages from disk using the ReadFromDiskFn
+func (logBuffer *LogBuffer) readMessagesFromDisk(startOffset int64, maxMessages int, maxBytes int, highWaterMark int64) (
+	messages []*filer_pb.LogEntry,
+	nextOffset int64,
+	highWaterMark2 int64,
+	endOfPartition bool,
+	err error,
+) {
+	if logBuffer.ReadFromDiskFn == nil {
+		return nil, startOffset, highWaterMark, true,
+			fmt.Errorf("no disk read function configured")
+	}
+
+	messages = make([]*filer_pb.LogEntry, 0, maxMessages)
+	nextOffset = startOffset
+	totalBytes := 0
+
+	// Use a simple callback to collect messages
+	collectFn := func(logEntry *filer_pb.LogEntry) (bool, error) {
+		// Check limits
+		if len(messages) >= maxMessages {
+			return true, nil // Done
+		}
+
+		entrySize := 4 + len(logEntry.Data) + len(logEntry.Key)
+		if totalBytes+entrySize > maxBytes {
+			return true, nil // Done
+		}
+
+		// Only include messages at or after startOffset
+		if logEntry.Offset >= startOffset {
+			messages = append(messages, logEntry)
+			totalBytes += entrySize
+			nextOffset = logEntry.Offset + 1
+		}
+
+		return false, nil // Continue
+	}
+
+	// Read from disk
+	startPos := NewMessagePositionFromOffset(startOffset)
+	_, isDone, err := logBuffer.ReadFromDiskFn(startPos, 0, collectFn)
+
+	if err != nil {
+		glog.Warningf("[StatelessRead] Disk read error: %v", err)
+		return nil, startOffset, highWaterMark, false, err
+	}
+
+	glog.V(4).Infof("[StatelessRead] Read %d messages from disk, nextOffset=%d, isDone=%v",
+		len(messages), nextOffset, isDone)
+
+	// If we read from disk and got no messages, and isDone is true, we're at the end
+	endOfPartition = isDone && len(messages) == 0
+
+	return messages, nextOffset, highWaterMark, endOfPartition, nil
+}
+
+// GetHighWaterMark returns the highest offset available in this partition
+// This is a lightweight operation for clients to check partition state
+func (logBuffer *LogBuffer) GetHighWaterMark() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+	return logBuffer.offset
+}
+
+// GetLogStartOffset returns the earliest offset available (either in memory or on disk)
+// This is useful for clients to know the valid offset range
+func (logBuffer *LogBuffer) GetLogStartOffset() int64 {
+	logBuffer.RLock()
+	defer logBuffer.RUnlock()
+
+	// Check if we have offset information
+	if !logBuffer.hasOffsets {
+		return 0
+	}
+
+	// Return the current buffer start offset - this is the earliest offset in memory RIGHT NOW
+	// For stateless fetch, we only return what's currently available in memory
+	// We don't check prevBuffers because they may be stale or getting flushed
+	return logBuffer.bufferStartOffset
+}
+
+// WaitForDataWithTimeout waits up to maxWaitMs for data to be available at startOffset
+// Returns true if data became available, false if timeout
+// This allows "long poll" behavior for real-time consumers
+func (logBuffer *LogBuffer) WaitForDataWithTimeout(startOffset int64, maxWaitMs int) bool {
+	if maxWaitMs <= 0 {
+		return false
+	}
+
+	timeout := time.NewTimer(time.Duration(maxWaitMs) * time.Millisecond)
+	defer timeout.Stop()
+
+	// Register for notifications
+	notifyChan := logBuffer.RegisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+	defer logBuffer.UnregisterSubscriber(fmt.Sprintf("fetch-%d", startOffset))
+
+	// Check if data is already available
+	logBuffer.RLock()
+	currentEnd := logBuffer.offset
+	logBuffer.RUnlock()
+
+	if currentEnd >= startOffset {
+		return true
+	}
+
+	// Wait for notification or timeout
+	select {
+	case <-notifyChan:
+		// Data might be available now
+		logBuffer.RLock()
+		currentEnd := logBuffer.offset
+		logBuffer.RUnlock()
+		return currentEnd >= startOffset
+	case <-timeout.C:
+		return false
+	}
+}

weed/util/log_buffer/log_read_stateless_test.go

@@ -0,0 +1,372 @@
+package log_buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestReadMessagesAtOffset_EmptyBuffer(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0 // Empty buffer
+
+	messages, nextOffset, hwm, endOfPartition, err := lb.ReadMessagesAtOffset(100, 10, 1024)
+
+	// Reading from future offset (100) when buffer is at 0
+	// Should return empty, no error
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages, got %d", len(messages))
+	}
+	if nextOffset != 100 {
+		t.Errorf("Expected nextOffset=100, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+	if hwm != 0 {
+		t.Errorf("Expected highWaterMark=0, got %d", hwm)
+	}
+}
+
+func TestReadMessagesAtOffset_SingleMessage(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add a message
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key1"),
+		Data:   []byte("value1"),
+		Offset: 0,
+	}
+	lb.AddLogEntryToBuffer(entry)
+
+	// Read from offset 0
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 1024)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 1 {
+		t.Errorf("Expected 1 message, got %d", len(messages))
+	}
+	if nextOffset != 1 {
+		t.Errorf("Expected nextOffset=1, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true after reading all messages")
+	}
+	if messages[0].Offset != 0 {
+		t.Errorf("Expected message offset=0, got %d", messages[0].Offset)
+	}
+	if string(messages[0].Key) != "key1" {
+		t.Errorf("Expected key='key1', got '%s'", string(messages[0].Key))
+	}
+}
+
+func TestReadMessagesAtOffset_MultipleMessages(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Read from offset 0, max 3 messages
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(0, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 3 {
+		t.Errorf("Expected nextOffset=3, got %d", nextOffset)
+	}
+
+	// Verify offsets are sequential
+	for i, msg := range messages {
+		if msg.Offset != int64(i) {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, i, msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_StartFromMiddle(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 10 messages (0-9)
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Read from offset 5
+	messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(5, 3, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+	if len(messages) != 3 {
+		t.Errorf("Expected 3 messages, got %d", len(messages))
+	}
+	if nextOffset != 8 {
+		t.Errorf("Expected nextOffset=8, got %d", nextOffset)
+	}
+
+	// Verify we got messages 5, 6, 7
+	expectedOffsets := []int64{5, 6, 7}
+	for i, msg := range messages {
+		if msg.Offset != expectedOffsets[i] {
+			t.Errorf("Message %d: expected offset=%d, got %d", i, expectedOffsets[i], msg.Offset)
+		}
+	}
+}
+
+func TestReadMessagesAtOffset_MaxBytesLimit(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add messages with 100 bytes each
+	for i := 0; i < 10; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   make([]byte, 100), // 100 bytes
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Request with max 250 bytes (should get ~2 messages)
+	messages, _, _, _, err := lb.ReadMessagesAtOffset(0, 100, 250)
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	// Should get at least 1 message, but likely 2
+	if len(messages) == 0 {
+		t.Error("Expected at least 1 message")
+	}
+	if len(messages) > 3 {
+		t.Errorf("Expected max 3 messages with 250 byte limit, got %d", len(messages))
+	}
+}
+
+func TestReadMessagesAtOffset_ConcurrentReads(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 100 messages
+	for i := 0; i < 100; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Start 10 concurrent readers at different offsets
+	done := make(chan bool, 10)
+
+	for reader := 0; reader < 10; reader++ {
+		startOffset := int64(reader * 10)
+		go func(offset int64) {
+			messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(offset, 5, 10240)
+
+			if err != nil {
+				t.Errorf("Reader at offset %d: unexpected error: %v", offset, err)
+			}
+			if len(messages) != 5 {
+				t.Errorf("Reader at offset %d: expected 5 messages, got %d", offset, len(messages))
+			}
+			if nextOffset != offset+5 {
+				t.Errorf("Reader at offset %d: expected nextOffset=%d, got %d", offset, offset+5, nextOffset)
+			}
+
+			// Verify sequential offsets
+			for i, msg := range messages {
+				expectedOffset := offset + int64(i)
+				if msg.Offset != expectedOffset {
+					t.Errorf("Reader at offset %d: message %d has offset %d, expected %d",
+						offset, i, msg.Offset, expectedOffset)
+				}
+			}
+
+			done <- true
+		}(startOffset)
+	}
+
+	// Wait for all readers
+	for i := 0; i < 10; i++ {
+		<-done
+	}
+}
+
+func TestReadMessagesAtOffset_FutureOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add 5 messages (0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// Try to read from offset 10 (future)
+	messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(10, 10, 10240)
+
+	if err != nil {
+		t.Errorf("Expected no error for future offset, got %v", err)
+	}
+	if len(messages) != 0 {
+		t.Errorf("Expected 0 messages for future offset, got %d", len(messages))
+	}
+	if nextOffset != 10 {
+		t.Errorf("Expected nextOffset=10, got %d", nextOffset)
+	}
+	if !endOfPartition {
+		t.Error("Expected endOfPartition=true for future offset")
+	}
+}
+
+func TestWaitForDataWithTimeout_DataAvailable(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Add message at offset 0
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	lb.AddLogEntryToBuffer(entry)
+
+	// Wait for data at offset 0 (should return immediately)
+	dataAvailable := lb.WaitForDataWithTimeout(0, 100)
+
+	if !dataAvailable {
+		t.Error("Expected data to be available at offset 0")
+	}
+}
+
+func TestWaitForDataWithTimeout_NoData(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 0
+	lb.offset = 0
+
+	// Don't add any messages, wait for offset 10
+
+	// Wait for data at offset 10 with short timeout
+	start := time.Now()
+	dataAvailable := lb.WaitForDataWithTimeout(10, 50)
+	elapsed := time.Since(start)
+
+	if dataAvailable {
+		t.Error("Expected no data to be available")
+	}
+	// Note: Actual wait time may be shorter if subscriber mechanism
+	// returns immediately. Just verify no data was returned.
+	t.Logf("Waited %v for timeout", elapsed)
+}
+
+func TestWaitForDataWithTimeout_DataArrives(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Start waiting in background
+	done := make(chan bool)
+	var dataAvailable bool
+
+	go func() {
+		dataAvailable = lb.WaitForDataWithTimeout(0, 500)
+		done <- true
+	}()
+
+	// Add data after 50ms
+	time.Sleep(50 * time.Millisecond)
+	entry := &filer_pb.LogEntry{
+		TsNs:   time.Now().UnixNano(),
+		Key:    []byte("key"),
+		Data:   []byte("value"),
+		Offset: 0,
+	}
+	lb.AddLogEntryToBuffer(entry)
+
+	// Wait for result
+	<-done
+
+	if !dataAvailable {
+		t.Error("Expected data to become available after being added")
+	}
+}
+
+func TestGetHighWaterMark(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+
+	// Initially should be 0
+	hwm := lb.GetHighWaterMark()
+	if hwm != 0 {
+		t.Errorf("Expected initial HWM=0, got %d", hwm)
+	}
+
+	// Add messages (offsets 0-4)
+	for i := 0; i < 5; i++ {
+		entry := &filer_pb.LogEntry{
+			TsNs:   time.Now().UnixNano(),
+			Key:    []byte("key"),
+			Data:   []byte("value"),
+			Offset: int64(i),
+		}
+		lb.AddLogEntryToBuffer(entry)
+	}
+
+	// HWM should be 5 (next offset to write, not last written offset)
+	// This matches Kafka semantics where HWM = last offset + 1
+	hwm = lb.GetHighWaterMark()
+	if hwm != 5 {
+		t.Errorf("Expected HWM=5 after adding 5 messages (0-4), got %d", hwm)
+	}
+}
+
+func TestGetLogStartOffset(t *testing.T) {
+	lb := NewLogBuffer("test", time.Hour, nil, nil, func() {})
+	lb.hasOffsets = true
+	lb.bufferStartOffset = 10
+
+	lso := lb.GetLogStartOffset()
+	if lso != 10 {
+		t.Errorf("Expected LSO=10, got %d", lso)
+	}
+}