fix kafka gateway and consumer group e2e flakes (#9129)

* fix(test): reduce kafka gateway and consumer group flakes

* fix(kafka): make broker health-check backoff respect context

Replace time.Sleep in the retry loop with a select on bc.ctx.Done() and
time.After so the backoff is interruptible during shutdown, per review
feedback on PR #9129.

* fix(kafka): guard broker HealthCheck against nil client

Return the same "broker client not connected" error used by the other
exported BrokerClient methods instead of panicking on a partially
initialized client, per CodeRabbit review feedback on PR #9129.

weed/mq/kafka/integration/broker_client.go

@@ -21,6 +21,12 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
+const (
+	brokerHealthCheckTimeout  = 5 * time.Second
+	brokerHealthCheckAttempts = 3
+	brokerHealthCheckBackoff  = 250 * time.Millisecond
+)
+
 // NewBrokerClientWithFilerAccessor creates a client with a shared filer accessor
 func NewBrokerClientWithFilerAccessor(brokerAddress string, filerClientAccessor *filer_client.FilerClientAccessor) (*BrokerClient, error) {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -98,17 +104,30 @@ func (bc *BrokerClient) Close() error {
 
 // HealthCheck verifies the broker connection is working
 func (bc *BrokerClient) HealthCheck() error {
-	// Create a timeout context for health check
-	ctx, cancel := context.WithTimeout(bc.ctx, 2*time.Second)
-	defer cancel()
-
-	// Try to list topics as a health check
-	_, err := bc.client.ListTopics(ctx, &mq_pb.ListTopicsRequest{})
-	if err != nil {
-		return fmt.Errorf("broker health check failed: %v", err)
+	if bc.client == nil {
+		return fmt.Errorf("broker client not connected")
 	}
 
-	return nil
+	var lastErr error
+	for attempt := 1; attempt <= brokerHealthCheckAttempts; attempt++ {
+		ctx, cancel := context.WithTimeout(bc.ctx, brokerHealthCheckTimeout)
+		_, err := bc.client.FindBrokerLeader(ctx, &mq_pb.FindBrokerLeaderRequest{})
+		cancel()
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if attempt == brokerHealthCheckAttempts {
+			break
+		}
+		select {
+		case <-bc.ctx.Done():
+			return bc.ctx.Err()
+		case <-time.After(time.Duration(attempt) * brokerHealthCheckBackoff):
+		}
+	}
+
+	return fmt.Errorf("broker health check failed after %d attempts: %v", brokerHealthCheckAttempts, lastErr)
 }
 
 // GetPartitionRangeInfo gets comprehensive range information from SeaweedMQ broker's native range manager