mq(kafka): Phase 3 Step 1 - Consumer Group Foundation

- Implement comprehensive consumer group coordinator with state management
- Add JoinGroup API (key 11) for consumer group membership
- Add SyncGroup API (key 14) for partition assignment coordination
- Create Range and RoundRobin assignment strategies
- Support consumer group lifecycle: Empty -> PreparingRebalance -> CompletingRebalance -> Stable
- Add automatic member cleanup and expired session handling
- Comprehensive test coverage for consumer groups, assignment strategies
- Update ApiVersions to advertise 9 APIs total (was 7)
- All existing integration tests pass with new consumer group support

This provides the foundation for distributed Kafka consumers with automatic
partition rebalancing and group coordination, compatible with standard Kafka clients.
This commit is contained in:
chrislu committed 2025-09-10 13:34:28 -07:00
1 parent 5aee693eac
commit d415911943
16 files changed
+2374 -317

No files matched your search

+23 -23
View File
@@ -110,7 +110,7 @@ func testSeaweedMQTopicLifecycle(t *testing.T, addr string) {
// Test CreateTopics request
topicName := "seaweedmq-test-topic"
createReq := buildCreateTopicsRequestCustom(topicName)
_, err = conn.Write(createReq)
if err != nil {
t.Fatalf("Failed to send CreateTopics: %v", err)
@@ -143,7 +143,7 @@ func testSeaweedMQTopicLifecycle(t *testing.T, addr string) {
func testSeaweedMQProduceConsume(t *testing.T, addr string) {
// This would be a more comprehensive test in a full implementation
// For now, just test that Produce requests are handled
conn, err := net.DialTimeout("tcp", addr, 5*time.Second)
if err != nil {
t.Fatalf("Failed to connect: %v", err)
@@ -174,65 +174,65 @@ func testSeaweedMQProduceConsume(t *testing.T, addr string) {
// TODO: Send a Produce request and verify it works with SeaweedMQ
// This would require building a proper Kafka Produce request
t.Logf("SeaweedMQ produce/consume test placeholder completed")
}
// buildCreateTopicsRequestCustom creates a CreateTopics request for a specific topic
func buildCreateTopicsRequestCustom(topicName string) []byte {
clientID := "seaweedmq-test-client"
// Approximate message size
messageSize := 2 + 2 + 4 + 2 + len(clientID) + 4 + 4 + 2 + len(topicName) + 4 + 2 + 4 + 4
request := make([]byte, 0, messageSize+4)
// Message size placeholder
sizePos := len(request)
request = append(request, 0, 0, 0, 0)
// API key (CreateTopics = 19)
request = append(request, 0, 19)
// API version
request = append(request, 0, 4)
// Correlation ID
request = append(request, 0, 0, 0x30, 0x42) // 12354
// Client ID
request = append(request, 0, byte(len(clientID)))
request = append(request, []byte(clientID)...)
// Timeout (5000ms)
request = append(request, 0, 0, 0x13, 0x88)
// Topics count (1)
request = append(request, 0, 0, 0, 1)
// Topic name
request = append(request, 0, byte(len(topicName)))
request = append(request, []byte(topicName)...)
// Num partitions (1)
request = append(request, 0, 0, 0, 1)
// Replication factor (1)
request = append(request, 0, 1)
// Configs count (0)
request = append(request, 0, 0, 0, 0)
// Topic timeout (5000ms)
request = append(request, 0, 0, 0x13, 0x88)
// Fix message size
actualSize := len(request) - 4
request[sizePos] = byte(actualSize >> 24)
request[sizePos+1] = byte(actualSize >> 16)
request[sizePos+2] = byte(actualSize >> 8)
request[sizePos+3] = byte(actualSize)
return request
}
@@ -285,8 +285,8 @@ func TestSeaweedMQGateway_ModeSelection(t *testing.T) {
// TestSeaweedMQGateway_ConfigValidation tests configuration validation
func TestSeaweedMQGateway_ConfigValidation(t *testing.T) {
testCases := []struct {
name string
options gateway.Options
name string
options gateway.Options
shouldWork bool
}{
{
@@ -321,11 +321,11 @@ func TestSeaweedMQGateway_ConfigValidation(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
server := gateway.NewServer(tc.options)
err := server.Start()
if tc.shouldWork && err != nil {
t.Errorf("Expected config to work, got error: %v", err)
}
if err == nil {
server.Close()
t.Logf("Config test passed for %s", tc.name)