Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-10-30 08:50:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

verify produced messages are consumed

Browse Source
This commit is contained in:
chrislu committed 2025-10-15 09:58:31 -07:00
1 parent e7747a7572
commit f4a018e731
4 files changed
+323 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/kafka/kafka-client-loadtest/cmd/loadtest/main.go
+36 -4
View File
@@ -22,6 +22,7 @@ import (
"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/metrics"
"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/producer"
"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/schema"
"github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest/internal/tracker"
)
var (
@@ -143,6 +144,9 @@ func main() {
func runProducerTest(ctx context.Context, cfg *config.Config, collector *metrics.Collector, wg *sync.WaitGroup) error {
log.Printf("Starting producer-only test with %d producers", cfg.Producers.Count)
// Create record tracker (nil for now as producer-only test doesn't need comparison)
recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl")
errChan := make(chan error, cfg.Producers.Count)
for i := 0; i < cfg.Producers.Count; i++ {
@@ -150,7 +154,7 @@ func runProducerTest(ctx context.Context, cfg *config.Config, collector *metrics
go func(id int) {
defer wg.Done()
prod, err := producer.New(cfg, collector, id)
prod, err := producer.New(cfg, collector, id, recordTracker)
if err != nil {
log.Printf("Failed to create producer %d: %v", id, err)
errChan <- err
@@ -179,6 +183,9 @@ func runProducerTest(ctx context.Context, cfg *config.Config, collector *metrics
func runConsumerTest(ctx context.Context, cfg *config.Config, collector *metrics.Collector, wg *sync.WaitGroup) error {
log.Printf("Starting consumer-only test with %d consumers", cfg.Consumers.Count)
// Create record tracker
recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl")
errChan := make(chan error, cfg.Consumers.Count)
for i := 0; i < cfg.Consumers.Count; i++ {
@@ -186,7 +193,7 @@ func runConsumerTest(ctx context.Context, cfg *config.Config, collector *metrics
go func(id int) {
defer wg.Done()
cons, err := consumer.New(cfg, collector, id)
cons, err := consumer.New(cfg, collector, id, recordTracker)
if err != nil {
log.Printf("Failed to create consumer %d: %v", id, err)
errChan <- err
@@ -206,6 +213,9 @@ func runComprehensiveTest(ctx context.Context, cancel context.CancelFunc, cfg *c
log.Printf("Starting comprehensive test with %d producers and %d consumers",
cfg.Producers.Count, cfg.Consumers.Count)
// Create record tracker
recordTracker := tracker.NewTracker("/test-results/produced.jsonl", "/test-results/consumed.jsonl")
errChan := make(chan error, cfg.Producers.Count)
// Create separate contexts for producers and consumers
@@ -218,7 +228,7 @@ func runComprehensiveTest(ctx context.Context, cancel context.CancelFunc, cfg *c
go func(id int) {
defer wg.Done()
prod, err := producer.New(cfg, collector, id)
prod, err := producer.New(cfg, collector, id, recordTracker)
if err != nil {
log.Printf("Failed to create producer %d: %v", id, err)
errChan <- err
@@ -244,7 +254,7 @@ func runComprehensiveTest(ctx context.Context, cancel context.CancelFunc, cfg *c
go func(id int) {
defer wg.Done()
cons, err := consumer.New(cfg, collector, id)
cons, err := consumer.New(cfg, collector, id, recordTracker)
if err != nil {
log.Printf("Failed to create consumer %d: %v", id, err)
return
@@ -304,6 +314,28 @@ func runComprehensiveTest(ctx context.Context, cancel context.CancelFunc, cfg *c
}()
}
// Wait for all producer and consumer goroutines to complete
log.Printf("Waiting for all producers and consumers to complete...")
wg.Wait()
log.Printf("All producers and consumers completed, starting verification...")
// Save produced and consumed records
log.Printf("Saving produced records...")
if err := recordTracker.SaveProduced(); err != nil {
log.Printf("Failed to save produced records: %v", err)
}
log.Printf("Saving consumed records...")
if err := recordTracker.SaveConsumed(); err != nil {
log.Printf("Failed to save consumed records: %v", err)
}
// Compare records
log.Printf("Comparing produced vs consumed records...")
result := recordTracker.Compare()
result.PrintSummary()
log.Printf("Verification complete!")
return nil
}