Files
seaweedfs/weed/mq/kafka/integration/agent_client.go
T
chrislu d415911943 mq(kafka): Phase 3 Step 1 - Consumer Group Foundation
- Implement comprehensive consumer group coordinator with state management
- Add JoinGroup API (key 11) for consumer group membership
- Add SyncGroup API (key 14) for partition assignment coordination
- Create Range and RoundRobin assignment strategies
- Support consumer group lifecycle: Empty -> PreparingRebalance -> CompletingRebalance -> Stable
- Add automatic member cleanup and expired session handling
- Comprehensive test coverage for consumer groups, assignment strategies
- Update ApiVersions to advertise 9 APIs total (was 7)
- All existing integration tests pass with new consumer group support

This provides the foundation for distributed Kafka consumers with automatic
partition rebalancing and group coordination, compatible with standard Kafka clients.
2025-09-10 13:34:28 -07:00

404 lines
11 KiB
Go

package integration
import (
"context"
"fmt"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/offset"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
)
// AgentClient wraps the SeaweedMQ Agent gRPC client for Kafka gateway integration
type AgentClient struct {
agentAddress string
conn *grpc.ClientConn
client mq_agent_pb.SeaweedMessagingAgentClient
// Publisher sessions: topic-partition -> session info
publishersLock sync.RWMutex
publishers map[string]*PublisherSession
// Subscriber sessions for offset tracking
subscribersLock sync.RWMutex
subscribers map[string]*SubscriberSession
ctx context.Context
cancel context.CancelFunc
}
// PublisherSession tracks a publishing session to SeaweedMQ
type PublisherSession struct {
SessionID int64
Topic string
Partition int32
Stream mq_agent_pb.SeaweedMessagingAgent_PublishRecordClient
RecordType *schema_pb.RecordType
LastSequence int64
}
// SubscriberSession tracks a subscription for offset management
type SubscriberSession struct {
Topic string
Partition int32
Stream mq_agent_pb.SeaweedMessagingAgent_SubscribeRecordClient
OffsetLedger *offset.Ledger // Still use for Kafka offset translation
}
// NewAgentClient creates a new SeaweedMQ Agent client
func NewAgentClient(agentAddress string) (*AgentClient, error) {
ctx, cancel := context.WithCancel(context.Background())
conn, err := grpc.DialContext(ctx, agentAddress,
grpc.WithTransportCredentials(insecure.NewCredentials()),
// Don't block - fail fast for invalid addresses
)
if err != nil {
cancel()
return nil, fmt.Errorf("failed to connect to agent %s: %v", agentAddress, err)
}
client := mq_agent_pb.NewSeaweedMessagingAgentClient(conn)
return &AgentClient{
agentAddress: agentAddress,
conn: conn,
client: client,
publishers: make(map[string]*PublisherSession),
subscribers: make(map[string]*SubscriberSession),
ctx: ctx,
cancel: cancel,
}, nil
}
// Close shuts down the agent client and all sessions
func (ac *AgentClient) Close() error {
ac.cancel()
// Close all publisher sessions
ac.publishersLock.Lock()
for key, session := range ac.publishers {
ac.closePublishSessionLocked(session.SessionID)
delete(ac.publishers, key)
}
ac.publishersLock.Unlock()
// Close all subscriber sessions
ac.subscribersLock.Lock()
for key, session := range ac.subscribers {
if session.Stream != nil {
session.Stream.CloseSend()
}
delete(ac.subscribers, key)
}
ac.subscribersLock.Unlock()
return ac.conn.Close()
}
// GetOrCreatePublisher gets or creates a publisher session for a topic-partition
func (ac *AgentClient) GetOrCreatePublisher(topic string, partition int32) (*PublisherSession, error) {
key := fmt.Sprintf("%s-%d", topic, partition)
// Try to get existing publisher
ac.publishersLock.RLock()
if session, exists := ac.publishers[key]; exists {
ac.publishersLock.RUnlock()
return session, nil
}
ac.publishersLock.RUnlock()
// Create new publisher session
ac.publishersLock.Lock()
defer ac.publishersLock.Unlock()
// Double-check after acquiring write lock
if session, exists := ac.publishers[key]; exists {
return session, nil
}
// Create the session
session, err := ac.createPublishSession(topic, partition)
if err != nil {
return nil, err
}
ac.publishers[key] = session
return session, nil
}
// createPublishSession creates a new publishing session with SeaweedMQ Agent
func (ac *AgentClient) createPublishSession(topic string, partition int32) (*PublisherSession, error) {
// Create a basic record type for Kafka messages
recordType := &schema_pb.RecordType{
Fields: []*schema_pb.Field{
{
Name: "key",
FieldIndex: 0,
Type: &schema_pb.Type{
Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_BYTES},
},
IsRequired: false,
IsRepeated: false,
},
{
Name: "value",
FieldIndex: 1,
Type: &schema_pb.Type{
Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_BYTES},
},
IsRequired: true,
IsRepeated: false,
},
{
Name: "timestamp",
FieldIndex: 2,
Type: &schema_pb.Type{
Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_TIMESTAMP},
},
IsRequired: false,
IsRepeated: false,
},
},
}
// Start publish session
startReq := &mq_agent_pb.StartPublishSessionRequest{
Topic: &schema_pb.Topic{
Namespace: "kafka", // Use "kafka" namespace for Kafka messages
Name: topic,
},
PartitionCount: 1, // For Phase 2, use single partition
RecordType: recordType,
PublisherName: "kafka-gateway",
}
startResp, err := ac.client.StartPublishSession(ac.ctx, startReq)
if err != nil {
return nil, fmt.Errorf("failed to start publish session: %v", err)
}
if startResp.Error != "" {
return nil, fmt.Errorf("publish session error: %s", startResp.Error)
}
// Create streaming connection
stream, err := ac.client.PublishRecord(ac.ctx)
if err != nil {
return nil, fmt.Errorf("failed to create publish stream: %v", err)
}
session := &PublisherSession{
SessionID: startResp.SessionId,
Topic: topic,
Partition: partition,
Stream: stream,
RecordType: recordType,
}
return session, nil
}
// PublishRecord publishes a single record to SeaweedMQ
func (ac *AgentClient) PublishRecord(topic string, partition int32, key []byte, value []byte, timestamp int64) (int64, error) {
session, err := ac.GetOrCreatePublisher(topic, partition)
if err != nil {
return 0, err
}
// Convert to SeaweedMQ record format
record := &schema_pb.RecordValue{
Fields: map[string]*schema_pb.Value{
"key": {
Kind: &schema_pb.Value_BytesValue{BytesValue: key},
},
"value": {
Kind: &schema_pb.Value_BytesValue{BytesValue: value},
},
"timestamp": {
Kind: &schema_pb.Value_TimestampValue{
TimestampValue: &schema_pb.TimestampValue{
TimestampMicros: timestamp / 1000, // Convert nanoseconds to microseconds
IsUtc: true,
},
},
},
},
}
// Send publish request
req := &mq_agent_pb.PublishRecordRequest{
SessionId: session.SessionID,
Key: key,
Value: record,
}
if err := session.Stream.Send(req); err != nil {
return 0, fmt.Errorf("failed to send record: %v", err)
}
// Read acknowledgment (this is a streaming API, so we should read the response)
resp, err := session.Stream.Recv()
if err != nil {
return 0, fmt.Errorf("failed to receive ack: %v", err)
}
if resp.Error != "" {
return 0, fmt.Errorf("publish error: %s", resp.Error)
}
session.LastSequence = resp.AckSequence
return resp.AckSequence, nil
}
// GetOrCreateSubscriber gets or creates a subscriber for offset tracking
func (ac *AgentClient) GetOrCreateSubscriber(topic string, partition int32, startOffset int64) (*SubscriberSession, error) {
key := fmt.Sprintf("%s-%d", topic, partition)
ac.subscribersLock.RLock()
if session, exists := ac.subscribers[key]; exists {
ac.subscribersLock.RUnlock()
return session, nil
}
ac.subscribersLock.RUnlock()
// Create new subscriber session
ac.subscribersLock.Lock()
defer ac.subscribersLock.Unlock()
if session, exists := ac.subscribers[key]; exists {
return session, nil
}
session, err := ac.createSubscribeSession(topic, partition, startOffset)
if err != nil {
return nil, err
}
ac.subscribers[key] = session
return session, nil
}
// createSubscribeSession creates a subscriber session for reading messages
func (ac *AgentClient) createSubscribeSession(topic string, partition int32, startOffset int64) (*SubscriberSession, error) {
stream, err := ac.client.SubscribeRecord(ac.ctx)
if err != nil {
return nil, fmt.Errorf("failed to create subscribe stream: %v", err)
}
// Send initial subscribe request
initReq := &mq_agent_pb.SubscribeRecordRequest{
Init: &mq_agent_pb.SubscribeRecordRequest_InitSubscribeRecordRequest{
ConsumerGroup: "kafka-gateway",
ConsumerGroupInstanceId: fmt.Sprintf("kafka-gateway-%s-%d", topic, partition),
Topic: &schema_pb.Topic{
Namespace: "kafka",
Name: topic,
},
PartitionOffsets: []*schema_pb.PartitionOffset{
{
Partition: &schema_pb.Partition{
RingSize: 1024, // Standard ring size
RangeStart: 0,
RangeStop: 1023,
},
StartTsNs: startOffset, // Use offset as timestamp for now
},
},
OffsetType: schema_pb.OffsetType_EXACT_TS_NS,
MaxSubscribedPartitions: 1,
SlidingWindowSize: 10,
},
}
if err := stream.Send(initReq); err != nil {
return nil, fmt.Errorf("failed to send subscribe init: %v", err)
}
session := &SubscriberSession{
Topic: topic,
Partition: partition,
Stream: stream,
OffsetLedger: offset.NewLedger(), // Keep Kafka offset tracking
}
return session, nil
}
// ClosePublisher closes a specific publisher session
func (ac *AgentClient) ClosePublisher(topic string, partition int32) error {
key := fmt.Sprintf("%s-%d", topic, partition)
ac.publishersLock.Lock()
defer ac.publishersLock.Unlock()
session, exists := ac.publishers[key]
if !exists {
return nil // Already closed or never existed
}
err := ac.closePublishSessionLocked(session.SessionID)
delete(ac.publishers, key)
return err
}
// closePublishSessionLocked closes a publish session (must be called with lock held)
func (ac *AgentClient) closePublishSessionLocked(sessionID int64) error {
closeReq := &mq_agent_pb.ClosePublishSessionRequest{
SessionId: sessionID,
}
_, err := ac.client.ClosePublishSession(ac.ctx, closeReq)
return err
}
// HealthCheck verifies the agent connection is working
func (ac *AgentClient) HealthCheck() error {
// Create a timeout context for health check
ctx, cancel := context.WithTimeout(ac.ctx, 2*time.Second)
defer cancel()
// Try to start and immediately close a dummy session
req := &mq_agent_pb.StartPublishSessionRequest{
Topic: &schema_pb.Topic{
Namespace: "kafka",
Name: "_health_check",
},
PartitionCount: 1,
RecordType: &schema_pb.RecordType{
Fields: []*schema_pb.Field{
{
Name: "test",
FieldIndex: 0,
Type: &schema_pb.Type{
Kind: &schema_pb.Type_ScalarType{ScalarType: schema_pb.ScalarType_STRING},
},
},
},
},
PublisherName: "health-check",
}
resp, err := ac.client.StartPublishSession(ctx, req)
if err != nil {
return fmt.Errorf("health check failed: %v", err)
}
if resp.Error != "" {
return fmt.Errorf("health check error: %s", resp.Error)
}
// Close the health check session
closeReq := &mq_agent_pb.ClosePublishSessionRequest{
SessionId: resp.SessionId,
}
_, _ = ac.client.ClosePublishSession(ctx, closeReq)
return nil
}