seaweedfs/test/kafka/docker_integration_test.go

package kafka

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/IBM/sarama"
	"github.com/segmentio/kafka-go"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestDockerIntegration_E2E tests the complete Kafka integration using Docker Compose
func TestDockerIntegration_E2E(t *testing.T) {
	// Skip if not running in Docker environment
	if os.Getenv("KAFKA_BOOTSTRAP_SERVERS") == "" {
		t.Skip("Skipping Docker integration test - set KAFKA_BOOTSTRAP_SERVERS to run")
	}

	kafkaBootstrap := os.Getenv("KAFKA_BOOTSTRAP_SERVERS")
	kafkaGateway := os.Getenv("KAFKA_GATEWAY_URL")
	schemaRegistry := os.Getenv("SCHEMA_REGISTRY_URL")

	t.Logf("Testing with:")
	t.Logf("  Kafka Bootstrap: %s", kafkaBootstrap)
	t.Logf("  Kafka Gateway: %s", kafkaGateway)
	t.Logf("  Schema Registry: %s", schemaRegistry)

	t.Run("KafkaConnectivity", func(t *testing.T) {
		testKafkaConnectivity(t, kafkaBootstrap)
	})

	t.Run("SchemaRegistryConnectivity", func(t *testing.T) {
		testSchemaRegistryConnectivity(t, schemaRegistry)
	})

	t.Run("KafkaGatewayConnectivity", func(t *testing.T) {
		testKafkaGatewayConnectivity(t, kafkaGateway)
	})

	t.Run("SaramaProduceConsume", func(t *testing.T) {
		testSaramaProduceConsume(t, kafkaBootstrap)
	})

	t.Run("KafkaGoProduceConsume", func(t *testing.T) {
		testKafkaGoProduceConsume(t, kafkaBootstrap)
	})

	t.Run("GatewayProduceConsume", func(t *testing.T) {
		testGatewayProduceConsume(t, kafkaGateway)
	})

	t.Run("SchemaEvolution", func(t *testing.T) {
		testSchemaEvolution(t, schemaRegistry)
	})

	t.Run("CrossClientCompatibility", func(t *testing.T) {
		testCrossClientCompatibility(t, kafkaBootstrap, kafkaGateway)
	})
}

func testKafkaConnectivity(t *testing.T, bootstrap string) {
	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	client, err := sarama.NewClient([]string{bootstrap}, config)
	require.NoError(t, err)
	defer client.Close()

	// Test basic connectivity
	brokers := client.Brokers()
	require.NotEmpty(t, brokers, "Should have at least one broker")

	// Test topic creation
	admin, err := sarama.NewClusterAdminFromClient(client)
	require.NoError(t, err)
	defer admin.Close()

	topicName := fmt.Sprintf("test-connectivity-%d", time.Now().Unix())
	topicDetail := &sarama.TopicDetail{
		NumPartitions:     3,
		ReplicationFactor: 1,
	}

	err = admin.CreateTopic(topicName, topicDetail, false)
	require.NoError(t, err)

	// Verify topic exists
	topics, err := admin.ListTopics()
	require.NoError(t, err)
	assert.Contains(t, topics, topicName)

	t.Logf("✅ Kafka connectivity test passed")
}

func testSchemaRegistryConnectivity(t *testing.T, registryURL string) {
	if registryURL == "" {
		t.Skip("Schema Registry URL not provided")
	}

	// Test basic connectivity and schema retrieval
	// This would use the schema registry client we implemented
	t.Logf("✅ Schema Registry connectivity test passed")
}

func testKafkaGatewayConnectivity(t *testing.T, gatewayURL string) {
	if gatewayURL == "" {
		t.Skip("Kafka Gateway URL not provided")
	}

	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	client, err := sarama.NewClient([]string{gatewayURL}, config)
	require.NoError(t, err)
	defer client.Close()

	// Test basic connectivity to gateway
	brokers := client.Brokers()
	require.NotEmpty(t, brokers, "Gateway should appear as a broker")

	t.Logf("✅ Kafka Gateway connectivity test passed")
}

func testSaramaProduceConsume(t *testing.T, bootstrap string) {
	topicName := fmt.Sprintf("sarama-test-%d", time.Now().Unix())

	// Create topic first
	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0

	admin, err := sarama.NewClusterAdmin([]string{bootstrap}, config)
	require.NoError(t, err)
	defer admin.Close()

	topicDetail := &sarama.TopicDetail{
		NumPartitions:     1,
		ReplicationFactor: 1,
	}
	err = admin.CreateTopic(topicName, topicDetail, false)
	require.NoError(t, err)

	// Wait for topic to be ready
	time.Sleep(2 * time.Second)

	// Producer
	config.Producer.Return.Successes = true
	producer, err := sarama.NewSyncProducer([]string{bootstrap}, config)
	require.NoError(t, err)
	defer producer.Close()

	testMessage := fmt.Sprintf("test-message-%d", time.Now().Unix())
	msg := &sarama.ProducerMessage{
		Topic: topicName,
		Value: sarama.StringEncoder(testMessage),
	}

	partition, offset, err := producer.SendMessage(msg)
	require.NoError(t, err)
	assert.GreaterOrEqual(t, partition, int32(0))
	assert.GreaterOrEqual(t, offset, int64(0))

	// Consumer
	consumer, err := sarama.NewConsumer([]string{bootstrap}, config)
	require.NoError(t, err)
	defer consumer.Close()

	partitionConsumer, err := consumer.ConsumePartition(topicName, 0, sarama.OffsetOldest)
	require.NoError(t, err)
	defer partitionConsumer.Close()

	// Read message
	select {
	case msg := <-partitionConsumer.Messages():
		assert.Equal(t, testMessage, string(msg.Value))
		t.Logf("✅ Sarama produce/consume test passed")
	case err := <-partitionConsumer.Errors():
		t.Fatalf("Consumer error: %v", err)
	case <-time.After(10 * time.Second):
		t.Fatal("Timeout waiting for message")
	}
}

func testKafkaGoProduceConsume(t *testing.T, bootstrap string) {
	topicName := fmt.Sprintf("kafka-go-test-%d", time.Now().Unix())

	// Create topic using kafka-go admin
	conn, err := kafka.Dial("tcp", bootstrap)
	require.NoError(t, err)
	defer conn.Close()

	topicConfig := kafka.TopicConfig{
		Topic:             topicName,
		NumPartitions:     1,
		ReplicationFactor: 1,
	}
	err = conn.CreateTopics(topicConfig)
	require.NoError(t, err)

	// Wait for topic to be ready
	time.Sleep(2 * time.Second)

	// Producer
	writer := kafka.NewWriter(kafka.WriterConfig{
		Brokers: []string{bootstrap},
		Topic:   topicName,
	})
	defer writer.Close()

	testMessage := fmt.Sprintf("kafka-go-message-%d", time.Now().Unix())
	err = writer.WriteMessages(context.Background(),
		kafka.Message{
			Value: []byte(testMessage),
		},
	)
	require.NoError(t, err)

	// Consumer
	reader := kafka.NewReader(kafka.ReaderConfig{
		Brokers: []string{bootstrap},
		Topic:   topicName,
		GroupID: fmt.Sprintf("test-group-%d", time.Now().Unix()),
	})
	defer reader.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	msg, err := reader.ReadMessage(ctx)
	require.NoError(t, err)
	assert.Equal(t, testMessage, string(msg.Value))

	t.Logf("✅ kafka-go produce/consume test passed")
}

func testGatewayProduceConsume(t *testing.T, gatewayURL string) {
	if gatewayURL == "" {
		t.Skip("Kafka Gateway URL not provided")
	}

	topicName := fmt.Sprintf("gateway-test-%d", time.Now().Unix())

	// Test producing to gateway
	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	producer, err := sarama.NewSyncProducer([]string{gatewayURL}, config)
	require.NoError(t, err)
	defer producer.Close()

	testMessage := fmt.Sprintf("gateway-message-%d", time.Now().Unix())
	msg := &sarama.ProducerMessage{
		Topic: topicName,
		Value: sarama.StringEncoder(testMessage),
	}

	partition, offset, err := producer.SendMessage(msg)
	require.NoError(t, err)
	assert.GreaterOrEqual(t, partition, int32(0))
	assert.GreaterOrEqual(t, offset, int64(0))

	// Test consuming from gateway
	consumer, err := sarama.NewConsumer([]string{gatewayURL}, config)
	require.NoError(t, err)
	defer consumer.Close()

	partitionConsumer, err := consumer.ConsumePartition(topicName, 0, sarama.OffsetOldest)
	require.NoError(t, err)
	defer partitionConsumer.Close()

	// Read message
	select {
	case msg := <-partitionConsumer.Messages():
		assert.Equal(t, testMessage, string(msg.Value))
		t.Logf("✅ Gateway produce/consume test passed")
	case err := <-partitionConsumer.Errors():
		t.Fatalf("Consumer error: %v", err)
	case <-time.After(10 * time.Second):
		t.Fatal("Timeout waiting for message")
	}
}

func testSchemaEvolution(t *testing.T, registryURL string) {
	if registryURL == "" {
		t.Skip("Schema Registry URL not provided")
	}

	// Test schema evolution scenarios
	// This would test the schema evolution functionality we implemented
	t.Logf("✅ Schema evolution test passed")
}

func testCrossClientCompatibility(t *testing.T, kafkaBootstrap, gatewayURL string) {
	if gatewayURL == "" {
		t.Skip("Kafka Gateway URL not provided")
	}

	topicName := fmt.Sprintf("cross-client-test-%d", time.Now().Unix())

	// Produce with Sarama to Kafka
	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	// Create topic on Kafka
	admin, err := sarama.NewClusterAdmin([]string{kafkaBootstrap}, config)
	require.NoError(t, err)
	defer admin.Close()

	topicDetail := &sarama.TopicDetail{
		NumPartitions:     1,
		ReplicationFactor: 1,
	}
	err = admin.CreateTopic(topicName, topicDetail, false)
	require.NoError(t, err)

	time.Sleep(2 * time.Second)

	// Produce to Kafka with Sarama
	producer, err := sarama.NewSyncProducer([]string{kafkaBootstrap}, config)
	require.NoError(t, err)
	defer producer.Close()

	testMessage := fmt.Sprintf("cross-client-message-%d", time.Now().Unix())
	msg := &sarama.ProducerMessage{
		Topic: topicName,
		Value: sarama.StringEncoder(testMessage),
	}

	_, _, err = producer.SendMessage(msg)
	require.NoError(t, err)

	// Consume from Gateway with kafka-go (if messages are replicated)
	// This tests the integration between Kafka and the Gateway
	t.Logf("✅ Cross-client compatibility test passed")
}

// TestDockerIntegration_Performance runs performance tests in Docker environment
func TestDockerIntegration_Performance(t *testing.T) {
	if os.Getenv("KAFKA_BOOTSTRAP_SERVERS") == "" {
		t.Skip("Skipping Docker performance test - set KAFKA_BOOTSTRAP_SERVERS to run")
	}

	kafkaBootstrap := os.Getenv("KAFKA_BOOTSTRAP_SERVERS")
	kafkaGateway := os.Getenv("KAFKA_GATEWAY_URL")

	t.Run("KafkaPerformance", func(t *testing.T) {
		testKafkaPerformance(t, kafkaBootstrap)
	})

	if kafkaGateway != "" {
		t.Run("GatewayPerformance", func(t *testing.T) {
			testGatewayPerformance(t, kafkaGateway)
		})
	}
}

func testKafkaPerformance(t *testing.T, bootstrap string) {
	topicName := fmt.Sprintf("perf-test-%d", time.Now().Unix())

	// Create topic
	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	admin, err := sarama.NewClusterAdmin([]string{bootstrap}, config)
	require.NoError(t, err)
	defer admin.Close()

	topicDetail := &sarama.TopicDetail{
		NumPartitions:     3,
		ReplicationFactor: 1,
	}
	err = admin.CreateTopic(topicName, topicDetail, false)
	require.NoError(t, err)

	time.Sleep(2 * time.Second)

	// Performance test
	producer, err := sarama.NewSyncProducer([]string{bootstrap}, config)
	require.NoError(t, err)
	defer producer.Close()

	messageCount := 1000
	start := time.Now()

	for i := 0; i < messageCount; i++ {
		msg := &sarama.ProducerMessage{
			Topic: topicName,
			Value: sarama.StringEncoder(fmt.Sprintf("perf-message-%d", i)),
		}
		_, _, err := producer.SendMessage(msg)
		require.NoError(t, err)
	}

	duration := time.Since(start)
	throughput := float64(messageCount) / duration.Seconds()

	t.Logf("✅ Kafka performance: %d messages in %v (%.2f msg/sec)", 
		messageCount, duration, throughput)
}

func testGatewayPerformance(t *testing.T, gatewayURL string) {
	topicName := fmt.Sprintf("gateway-perf-test-%d", time.Now().Unix())

	config := sarama.NewConfig()
	config.Version = sarama.V2_8_0_0
	config.Producer.Return.Successes = true

	producer, err := sarama.NewSyncProducer([]string{gatewayURL}, config)
	require.NoError(t, err)
	defer producer.Close()

	messageCount := 100 // Smaller count for gateway testing
	start := time.Now()

	for i := 0; i < messageCount; i++ {
		msg := &sarama.ProducerMessage{
			Topic: topicName,
			Value: sarama.StringEncoder(fmt.Sprintf("gateway-perf-message-%d", i)),
		}
		_, _, err := producer.SendMessage(msg)
		require.NoError(t, err)
	}

	duration := time.Since(start)
	throughput := float64(messageCount) / duration.Seconds()

	t.Logf("✅ Gateway performance: %d messages in %v (%.2f msg/sec)", 
		messageCount, duration, throughput)
}