mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-12 17:40:43 +02:00
chrislu
00a672d12e
MAJOR ENHANCEMENT: Complete Docker-based integration testing infrastructure ## New Docker Compose Infrastructure: - docker-compose.yml: Complete multi-service setup with health checks - Apache Kafka + Zookeeper - Confluent Schema Registry - SeaweedFS full stack (Master, Volume, Filer, MQ Broker, MQ Agent) - Kafka Gateway service - Test setup and utility services ## Docker Services: - Dockerfile.kafka-gateway: Custom Kafka Gateway container - Dockerfile.test-setup: Schema registration and test data setup - kafka-gateway-start.sh: Service startup script with dependency waiting - wait-for-services.sh: Comprehensive service readiness verification ## Test Setup Utility: - cmd/setup/main.go: Automated schema registration utility - Registers User, UserEvent, and LogEntry Avro schemas - Handles service discovery and health checking ## Integration Tests: - docker_integration_test.go: Comprehensive Docker-based integration tests - Kafka connectivity and topic operations - Schema Registry integration - Kafka Gateway functionality - Sarama and kafka-go client compatibility - Cross-client message compatibility - Performance benchmarking ## Build and Test Infrastructure: - Makefile: 30+ targets for development and testing - setup, test-unit, test-integration, test-e2e - Performance testing and benchmarking - Individual service management - Debugging and monitoring tools - CI/CD integration targets ## Documentation: - README.md: Comprehensive documentation - Architecture overview and service descriptions - Quick start guide and development workflow - Troubleshooting and performance tuning - CI/CD integration examples ## Key Features: ✅ Complete service orchestration with health checks ✅ Automated schema registration and test data setup ✅ Multi-client compatibility testing (Sarama, kafka-go) ✅ Performance benchmarking and monitoring ✅ Development-friendly debugging tools ✅ CI/CD ready with proper cleanup ✅ Comprehensive documentation and examples ## Usage: make setup-schemas # Start all services and register schemas make test-e2e # Run end-to-end integration tests make clean # Clean up environment This provides a production-ready testing infrastructure that ensures Kafka Gateway compatibility with real Kafka ecosystems and validates schema registry integration in realistic deployment scenarios.
319 lines
8.5 KiB
YAML
319 lines
8.5 KiB
YAML
version: '3.8'
|
|
|
|
services:
|
|
# Zookeeper for Kafka
|
|
zookeeper:
|
|
image: confluentinc/cp-zookeeper:7.4.0
|
|
container_name: kafka-zookeeper
|
|
ports:
|
|
- "2181:2181"
|
|
environment:
|
|
ZOOKEEPER_CLIENT_PORT: 2181
|
|
ZOOKEEPER_TICK_TIME: 2000
|
|
healthcheck:
|
|
test: ["CMD", "nc", "-z", "localhost", "2181"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# Kafka Broker
|
|
kafka:
|
|
image: confluentinc/cp-kafka:7.4.0
|
|
container_name: kafka-broker
|
|
ports:
|
|
- "9092:9092"
|
|
- "29092:29092"
|
|
depends_on:
|
|
zookeeper:
|
|
condition: service_healthy
|
|
environment:
|
|
KAFKA_BROKER_ID: 1
|
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
|
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
|
|
KAFKA_NUM_PARTITIONS: 3
|
|
KAFKA_DEFAULT_REPLICATION_FACTOR: 1
|
|
healthcheck:
|
|
test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 30s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# Schema Registry
|
|
schema-registry:
|
|
image: confluentinc/cp-schema-registry:7.4.0
|
|
container_name: kafka-schema-registry
|
|
ports:
|
|
- "8081:8081"
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
environment:
|
|
SCHEMA_REGISTRY_HOST_NAME: schema-registry
|
|
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
|
|
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
|
|
SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
|
|
SCHEMA_REGISTRY_DEBUG: "true"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 20s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# SeaweedFS Master
|
|
seaweedfs-master:
|
|
image: chrislusf/seaweedfs:latest
|
|
container_name: seaweedfs-master
|
|
ports:
|
|
- "9333:9333"
|
|
- "19333:19333" # gRPC port
|
|
command:
|
|
- master
|
|
- -ip=seaweedfs-master
|
|
- -port=9333
|
|
- -port.grpc=19333
|
|
- -volumeSizeLimitMB=1024
|
|
- -defaultReplication=000
|
|
volumes:
|
|
- seaweedfs-master-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:9333/cluster/status"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# SeaweedFS Volume Server
|
|
seaweedfs-volume:
|
|
image: chrislusf/seaweedfs:latest
|
|
container_name: seaweedfs-volume
|
|
ports:
|
|
- "8080:8080"
|
|
- "18080:18080" # gRPC port
|
|
command:
|
|
- volume
|
|
- -mserver=seaweedfs-master:9333
|
|
- -ip=seaweedfs-volume
|
|
- -port=8080
|
|
- -port.grpc=18080
|
|
- -publicUrl=seaweedfs-volume:8080
|
|
- -preStopSeconds=1
|
|
depends_on:
|
|
seaweedfs-master:
|
|
condition: service_healthy
|
|
volumes:
|
|
- seaweedfs-volume-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/status"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# SeaweedFS Filer
|
|
seaweedfs-filer:
|
|
image: chrislusf/seaweedfs:latest
|
|
container_name: seaweedfs-filer
|
|
ports:
|
|
- "8888:8888"
|
|
- "18888:18888" # gRPC port
|
|
command:
|
|
- filer
|
|
- -master=seaweedfs-master:9333
|
|
- -ip=seaweedfs-filer
|
|
- -port=8888
|
|
- -port.grpc=18888
|
|
depends_on:
|
|
seaweedfs-master:
|
|
condition: service_healthy
|
|
seaweedfs-volume:
|
|
condition: service_healthy
|
|
volumes:
|
|
- seaweedfs-filer-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8888/"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 15s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# SeaweedFS MQ Broker
|
|
seaweedfs-mq-broker:
|
|
image: chrislusf/seaweedfs:latest
|
|
container_name: seaweedfs-mq-broker
|
|
ports:
|
|
- "17777:17777" # MQ Broker port
|
|
command:
|
|
- mq.broker
|
|
- -filer=seaweedfs-filer:8888
|
|
- -ip=seaweedfs-mq-broker
|
|
- -port=17777
|
|
depends_on:
|
|
seaweedfs-filer:
|
|
condition: service_healthy
|
|
volumes:
|
|
- seaweedfs-mq-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "nc", "-z", "localhost", "17777"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 20s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# SeaweedFS MQ Agent
|
|
seaweedfs-mq-agent:
|
|
image: chrislusf/seaweedfs:latest
|
|
container_name: seaweedfs-mq-agent
|
|
ports:
|
|
- "16777:16777" # MQ Agent port
|
|
command:
|
|
- mq.agent
|
|
- -filer=seaweedfs-filer:8888
|
|
- -mq.broker=seaweedfs-mq-broker:17777
|
|
- -ip=seaweedfs-mq-agent
|
|
- -port=16777
|
|
depends_on:
|
|
seaweedfs-mq-broker:
|
|
condition: service_healthy
|
|
volumes:
|
|
- seaweedfs-mq-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "nc", "-z", "localhost", "16777"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 25s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# Kafka Gateway (SeaweedFS with Kafka protocol)
|
|
kafka-gateway:
|
|
build:
|
|
context: ../.. # Build from project root
|
|
dockerfile: test/kafka/Dockerfile.kafka-gateway
|
|
container_name: kafka-gateway
|
|
ports:
|
|
- "9093:9093" # Kafka protocol port
|
|
depends_on:
|
|
seaweedfs-mq-agent:
|
|
condition: service_healthy
|
|
schema-registry:
|
|
condition: service_healthy
|
|
environment:
|
|
- SEAWEEDFS_FILER=seaweedfs-filer:8888
|
|
- SEAWEEDFS_MQ_BROKER=seaweedfs-mq-broker:17777
|
|
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
|
- KAFKA_PORT=9093
|
|
volumes:
|
|
- kafka-gateway-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "nc", "-z", "localhost", "9093"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 30s
|
|
networks:
|
|
- kafka-test-net
|
|
|
|
# Test Data Setup Service
|
|
test-setup:
|
|
build:
|
|
context: ../..
|
|
dockerfile: test/kafka/Dockerfile.test-setup
|
|
container_name: kafka-test-setup
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
schema-registry:
|
|
condition: service_healthy
|
|
kafka-gateway:
|
|
condition: service_healthy
|
|
environment:
|
|
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
|
|
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
|
- KAFKA_GATEWAY_URL=kafka-gateway:9093
|
|
networks:
|
|
- kafka-test-net
|
|
restart: "no" # Run once to set up test data
|
|
profiles:
|
|
- setup # Only start when explicitly requested
|
|
|
|
# Kafka Producer for Testing
|
|
kafka-producer:
|
|
image: confluentinc/cp-kafka:7.4.0
|
|
container_name: kafka-producer
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
schema-registry:
|
|
condition: service_healthy
|
|
environment:
|
|
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
|
|
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
|
networks:
|
|
- kafka-test-net
|
|
profiles:
|
|
- producer # Only start when explicitly requested
|
|
command: >
|
|
sh -c "
|
|
echo 'Creating test topics...';
|
|
kafka-topics --create --topic test-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
|
|
kafka-topics --create --topic avro-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
|
|
kafka-topics --create --topic schema-test --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --if-not-exists;
|
|
echo 'Topics created successfully';
|
|
kafka-topics --list --bootstrap-server kafka:29092;
|
|
"
|
|
|
|
# Kafka Consumer for Testing
|
|
kafka-consumer:
|
|
image: confluentinc/cp-kafka:7.4.0
|
|
container_name: kafka-consumer
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
environment:
|
|
- KAFKA_BOOTSTRAP_SERVERS=kafka:29092
|
|
networks:
|
|
- kafka-test-net
|
|
profiles:
|
|
- consumer # Only start when explicitly requested
|
|
command: >
|
|
kafka-console-consumer
|
|
--bootstrap-server kafka:29092
|
|
--topic test-topic
|
|
--from-beginning
|
|
--max-messages 10
|
|
|
|
volumes:
|
|
seaweedfs-master-data:
|
|
seaweedfs-volume-data:
|
|
seaweedfs-filer-data:
|
|
seaweedfs-mq-data:
|
|
kafka-gateway-data:
|
|
|
|
networks:
|
|
kafka-test-net:
|
|
driver: bridge
|
|
name: kafka-integration-test
|