Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-20 13:30:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add comprehensive Docker Compose setup for Kafka integration tests

Browse Source 
MAJOR ENHANCEMENT: Complete Docker-based integration testing infrastructure

## New Docker Compose Infrastructure:
- docker-compose.yml: Complete multi-service setup with health checks
  - Apache Kafka + Zookeeper
  - Confluent Schema Registry
  - SeaweedFS full stack (Master, Volume, Filer, MQ Broker, MQ Agent)
  - Kafka Gateway service
  - Test setup and utility services

## Docker Services:
- Dockerfile.kafka-gateway: Custom Kafka Gateway container
- Dockerfile.test-setup: Schema registration and test data setup
- kafka-gateway-start.sh: Service startup script with dependency waiting
- wait-for-services.sh: Comprehensive service readiness verification

## Test Setup Utility:
- cmd/setup/main.go: Automated schema registration utility
- Registers User, UserEvent, and LogEntry Avro schemas
- Handles service discovery and health checking

## Integration Tests:
- docker_integration_test.go: Comprehensive Docker-based integration tests
  - Kafka connectivity and topic operations
  - Schema Registry integration
  - Kafka Gateway functionality
  - Sarama and kafka-go client compatibility
  - Cross-client message compatibility
  - Performance benchmarking

## Build and Test Infrastructure:
- Makefile: 30+ targets for development and testing
  - setup, test-unit, test-integration, test-e2e
  - Performance testing and benchmarking
  - Individual service management
  - Debugging and monitoring tools
  - CI/CD integration targets

## Documentation:
- README.md: Comprehensive documentation
  - Architecture overview and service descriptions
  - Quick start guide and development workflow
  - Troubleshooting and performance tuning
  - CI/CD integration examples

## Key Features:
✅ Complete service orchestration with health checks
✅ Automated schema registration and test data setup
✅ Multi-client compatibility testing (Sarama, kafka-go)
✅ Performance benchmarking and monitoring
✅ Development-friendly debugging tools
✅ CI/CD ready with proper cleanup
✅ Comprehensive documentation and examples

## Usage:
make setup-schemas  # Start all services and register schemas
make test-e2e      # Run end-to-end integration tests
make clean         # Clean up environment

This provides a production-ready testing infrastructure that ensures
Kafka Gateway compatibility with real Kafka ecosystems and validates
schema registry integration in realistic deployment scenarios.
This commit is contained in:
chrislu
2025-09-12 08:46:03 -07:00
parent e70421bb81
commit 00a672d12e
9 changed files with 1739 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/kafka/docker-compose.yml
+318
View File
@@ -0,0 +1,318 @@
version: '3.8'
services:
# Zookeeper for Kafka
zookeeper:
image: confluentinc/cp-zookeeper:7.4.0
container_name: kafka-zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "2181"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
networks:
- kafka-test-net
# Kafka Broker
kafka:
image: confluentinc/cp-kafka:7.4.0
container_name: kafka-broker
ports:
- "9092:9092"
- "29092:29092"
depends_on:
zookeeper:
condition: service_healthy
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_NUM_PARTITIONS: 3
KAFKA_DEFAULT_REPLICATION_FACTOR: 1
healthcheck:
test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-test-net
# Schema Registry
schema-registry:
image: confluentinc/cp-schema-registry:7.4.0
container_name: kafka-schema-registry
ports:
- "8081:8081"
depends_on:
kafka:
condition: service_healthy
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
SCHEMA_REGISTRY_DEBUG: "true"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
interval: 10s
timeout: 5s
retries: 5
start_period: 20s
networks:
- kafka-test-net
# SeaweedFS Master
seaweedfs-master:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-master
ports:
- "9333:9333"
- "19333:19333" # gRPC port
command:
- master
- -ip=seaweedfs-master
- -port=9333
- -port.grpc=19333
- -volumeSizeLimitMB=1024
- -defaultReplication=000
volumes:
- seaweedfs-master-data:/data
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:9333/cluster/status"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
networks:
- kafka-test-net
# SeaweedFS Volume Server
seaweedfs-volume:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-volume
ports:
- "8080:8080"
- "18080:18080" # gRPC port
command:
- volume
- -mserver=seaweedfs-master:9333
- -ip=seaweedfs-volume
- -port=8080
- -port.grpc=18080
- -publicUrl=seaweedfs-volume:8080
- -preStopSeconds=1
depends_on:
seaweedfs-master:
condition: service_healthy
volumes:
- seaweedfs-volume-data:/data
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/status"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
networks:
- kafka-test-net
# SeaweedFS Filer
seaweedfs-filer:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-filer
ports:
- "8888:8888"
- "18888:18888" # gRPC port
command:
- filer
- -master=seaweedfs-master:9333
- -ip=seaweedfs-filer
- -port=8888
- -port.grpc=18888
depends_on:
seaweedfs-master:
condition: service_healthy
seaweedfs-volume:
condition: service_healthy
volumes:
- seaweedfs-filer-data:/data
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8888/"]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
networks:
- kafka-test-net
# SeaweedFS MQ Broker
seaweedfs-mq-broker:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-mq-broker
ports:
- "17777:17777" # MQ Broker port
command:
- mq.broker
- -filer=seaweedfs-filer:8888
- -ip=seaweedfs-mq-broker
- -port=17777
depends_on:
seaweedfs-filer:
condition: service_healthy
volumes:
- seaweedfs-mq-data:/data
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "17777"]
interval: 10s
timeout: 5s
retries: 3
start_period: 20s
networks:
- kafka-test-net
# SeaweedFS MQ Agent
seaweedfs-mq-agent:
image: chrislusf/seaweedfs:latest
container_name: seaweedfs-mq-agent
ports:
- "16777:16777" # MQ Agent port
command:
- mq.agent
- -filer=seaweedfs-filer:8888
- -mq.broker=seaweedfs-mq-broker:17777
- -ip=seaweedfs-mq-agent
- -port=16777
depends_on:
seaweedfs-mq-broker:
condition: service_healthy
volumes:
- seaweedfs-mq-data:/data
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "16777"]
interval: 10s
timeout: 5s
retries: 3
start_period: 25s
networks:
- kafka-test-net
# Kafka Gateway (SeaweedFS with Kafka protocol)
kafka-gateway:
build:
context: ../.. # Build from project root
dockerfile: test/kafka/Dockerfile.kafka-gateway
container_name: kafka-gateway
ports:
- "9093:9093" # Kafka protocol port
depends_on:
seaweedfs-mq-agent:
condition: service_healthy
schema-registry:
condition: service_healthy
environment:
- SEAWEEDFS_FILER=seaweedfs-filer:8888
- SEAWEEDFS_MQ_BROKER=seaweedfs-mq-broker:17777
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- KAFKA_PORT=9093
volumes:
- kafka-gateway-data:/data
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "9093"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-test-net
# Test Data Setup Service
test-setup:
build:
context: ../..
dockerfile: test/kafka/Dockerfile.test-setup
container_name: kafka-test-setup
depends_on:
kafka:
condition: service_healthy
schema-registry:
condition: service_healthy
kafka-gateway:
condition: service_healthy
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
- KAFKA_GATEWAY_URL=kafka-gateway:9093
networks:
- kafka-test-net
restart: "no" # Run once to set up test data
profiles:
- setup # Only start when explicitly requested
# Kafka Producer for Testing
kafka-producer:
image: confluentinc/cp-kafka:7.4.0
container_name: kafka-producer
depends_on:
kafka:
condition: service_healthy
schema-registry:
condition: service_healthy
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
networks:
- kafka-test-net
profiles:
- producer # Only start when explicitly requested
command: >
sh -c "
echo 'Creating test topics...';
kafka-topics --create --topic test-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
kafka-topics --create --topic avro-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
kafka-topics --create --topic schema-test --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --if-not-exists;
echo 'Topics created successfully';
kafka-topics --list --bootstrap-server kafka:29092;
"
# Kafka Consumer for Testing
kafka-consumer:
image: confluentinc/cp-kafka:7.4.0
container_name: kafka-consumer
depends_on:
kafka:
condition: service_healthy
environment:
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
networks:
- kafka-test-net
profiles:
- consumer # Only start when explicitly requested
command: >
kafka-console-consumer
--bootstrap-server kafka:29092
--topic test-topic
--from-beginning
--max-messages 10
volumes:
seaweedfs-master-data:
seaweedfs-volume-data:
seaweedfs-filer-data:
seaweedfs-mq-data:
kafka-gateway-data:
networks:
kafka-test-net:
driver: bridge
name: kafka-integration-test