Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-20 13:30:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

Phase 1: Implement SeaweedMQ record retrieval in GetStoredRecords

Browse Source 
Core SeaweedMQ Integration completed:

## Implementation
- Implement SeaweedMQHandler.GetStoredRecords() to retrieve actual records from SeaweedMQ
- Add SeaweedSMQRecord wrapper implementing offset.SMQRecord interface
- Wire Fetch API to use real SMQ records instead of synthetic batches
- Support both agent and broker client connections for record retrieval

## Key Features
- Proper Kafka offset mapping from SeaweedMQ records
- Respects maxRecords limit and batch size constraints
- Graceful error handling for missing topics/partitions
- High water mark boundary checking

## Tests
- Unit tests for SMQRecord interface compliance
- Edge case testing (empty topics, offset boundaries, limits)
- Integration with existing end-to-end Kafka tests
- Benchmark tests for record accessor performance

## Verification
- All integration tests pass
- E2E Sarama test shows 'Found X SMQ records' debug output
- GetStoredRecords now returns real data instead of TODO placeholder

Ready for Phase 2: CreateTopics protocol compliance
This commit is contained in:
chrislu
2025-09-13 13:12:32 -07:00
parent dec63c22e4
commit c9f3935e7b
6 changed files with 694 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/kafka/sarama_simple_test.go
+3 -6
View File
@@ -12,8 +12,7 @@ import (
func TestSaramaSimpleProducer(t *testing.T) {
// Start gateway with test mode
gatewayServer := gateway.NewTestServer(gateway.Options{
Listen: "127.0.0.1:0",
Listen: "127.0.0.1:0",
})
go func() {
@@ -88,8 +87,7 @@ func TestSaramaSimpleProducer(t *testing.T) {
func TestSaramaMinimalConfig(t *testing.T) {
// Start gateway with test mode
gatewayServer := gateway.NewTestServer(gateway.Options{
Listen: "127.0.0.1:0",
Listen: "127.0.0.1:0",
})
go func() {
@@ -151,8 +149,7 @@ func TestSaramaMinimalConfig(t *testing.T) {
func TestSaramaProduceConsume(t *testing.T) {
// Start gateway with test mode
gatewayServer := gateway.NewTestServer(gateway.Options{
Listen: "127.0.0.1:0",
Listen: "127.0.0.1:0",
})
go func() {
test/kafka/sarama_test.go
+2 -4
View File
@@ -14,8 +14,7 @@ import (
func TestSaramaCompatibility(t *testing.T) {
// Start gateway with test mode
gatewayServer := gateway.NewTestServer(gateway.Options{
Listen: "127.0.0.1:0",
Listen: "127.0.0.1:0",
})
go gatewayServer.Start()
@@ -266,8 +265,7 @@ func (h *SaramaConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSe
func TestSaramaMetadataOnly(t *testing.T) {
// Start gateway with test mode
gatewayServer := gateway.NewTestServer(gateway.Options{
Listen: "127.0.0.1:0",
Listen: "127.0.0.1:0",
})
go gatewayServer.Start()