seaweedfs/weed/storage/blockvol/test/ha_target.go

//go:build integration

package test

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

// HATarget extends Target with HA-specific admin HTTP endpoints.
type HATarget struct {
	*Target
	AdminPort   int
	ReplicaData int // replica receiver data port
	ReplicaCtrl int // replica receiver ctrl port
	RebuildPort int
	TPGID       int // ALUA target port group ID (0 = omit flag)
}

// StatusResp matches the JSON returned by GET /status.
type StatusResp struct {
	Path          string `json:"path"`
	Epoch         uint64 `json:"epoch"`
	Role          string `json:"role"`
	WALHeadLSN    uint64 `json:"wal_head_lsn"`
	CheckpointLSN uint64 `json:"checkpoint_lsn"`
	HasLease      bool   `json:"has_lease"`
	Healthy       bool   `json:"healthy"`
}

// NewHATarget creates an HATarget with the given ports.
func NewHATarget(node *Node, cfg TargetConfig, adminPort, replicaData, replicaCtrl, rebuildPort int) *HATarget {
	return &HATarget{
		Target:      NewTarget(node, cfg),
		AdminPort:   adminPort,
		ReplicaData: replicaData,
		ReplicaCtrl: replicaCtrl,
		RebuildPort: rebuildPort,
	}
}

// Start overrides Target.Start to add HA-specific flags.
func (h *HATarget) Start(ctx context.Context, create bool) error {
	// Remove old log
	h.node.Run(ctx, fmt.Sprintf("rm -f %s", h.logFile))

	args := fmt.Sprintf("-vol %s -addr :%d -iqn %s",
		h.volFile, h.config.Port, h.config.IQN)

	if create {
		h.node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal", h.volFile, h.volFile))
		args += fmt.Sprintf(" -create -size %s", h.config.VolSize)
	}

	if h.AdminPort > 0 {
		args += fmt.Sprintf(" -admin 0.0.0.0:%d", h.AdminPort)
	}
	if h.ReplicaData > 0 && h.ReplicaCtrl > 0 {
		args += fmt.Sprintf(" -replica-data :%d -replica-ctrl :%d", h.ReplicaData, h.ReplicaCtrl)
	}
	if h.RebuildPort > 0 {
		args += fmt.Sprintf(" -rebuild-listen :%d", h.RebuildPort)
	}
	if h.TPGID > 0 {
		args += fmt.Sprintf(" -tpg-id %d", h.TPGID)
	}

	cmd := fmt.Sprintf("setsid -f %s %s >%s 2>&1", h.binPath, args, h.logFile)
	_, stderr, code, err := h.node.Run(ctx, cmd)
	if err != nil || code != 0 {
		return fmt.Errorf("start ha target: code=%d stderr=%s err=%v", code, stderr, err)
	}

	if err := h.WaitForPort(ctx); err != nil {
		return err
	}

	// Also wait for admin port if configured
	if h.AdminPort > 0 {
		if err := h.waitForAdminPort(ctx); err != nil {
			return err
		}
	}

	// Discover PID by matching the unique volume file path (not binPath,
	// which is shared across primary/replica targets).
	stdout, _, _, _ := h.node.Run(ctx, fmt.Sprintf("ps -eo pid,args | grep '%s' | grep -v grep | awk '{print $1}'", h.volFile))
	pidStr := strings.TrimSpace(stdout)
	if idx := strings.IndexByte(pidStr, '\n'); idx > 0 {
		pidStr = pidStr[:idx]
	}
	pid := 0
	fmt.Sscanf(pidStr, "%d", &pid)
	if pid == 0 {
		return fmt.Errorf("find ha target PID: %q", pidStr)
	}
	h.pid = pid
	return nil
}

func (h *HATarget) waitForAdminPort(ctx context.Context) error {
	for {
		select {
		case <-ctx.Done():
			return fmt.Errorf("wait for admin port %d: %w", h.AdminPort, ctx.Err())
		default:
		}
		stdout, _, code, _ := h.node.Run(ctx, fmt.Sprintf("ss -tln | grep :%d", h.AdminPort))
		if code == 0 && strings.Contains(stdout, fmt.Sprintf(":%d", h.AdminPort)) {
			return nil
		}
		time.Sleep(200 * time.Millisecond)
	}
}

// adminAddr returns host:port for admin requests.
func (h *HATarget) adminAddr() string {
	host := h.node.Host
	if h.node.IsLocal {
		host = "127.0.0.1"
	}
	return fmt.Sprintf("%s:%d", host, h.AdminPort)
}

// curlPost executes a POST via curl on the node (works in WSL2 and remote).
// Returns HTTP status code, response body, and error.
func (h *HATarget) curlPost(ctx context.Context, path string, body interface{}) (int, string, error) {
	data, err := json.Marshal(body)
	if err != nil {
		return 0, "", err
	}
	cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' -X POST -H 'Content-Type: application/json' -d '%s' http://127.0.0.1:%d%s 2>&1",
		string(data), h.AdminPort, path)
	stdout, _, code, err := h.node.Run(ctx, cmd)
	if err != nil || code != 0 {
		return 0, "", fmt.Errorf("curl POST %s: code=%d err=%v stdout=%s", path, code, err, stdout)
	}
	return parseCurlOutput(stdout)
}

// curlGet executes a GET via curl on the node.
func (h *HATarget) curlGet(ctx context.Context, path string) (int, string, error) {
	cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' http://127.0.0.1:%d%s 2>&1", h.AdminPort, path)
	stdout, _, code, err := h.node.Run(ctx, cmd)
	if err != nil || code != 0 {
		return 0, "", fmt.Errorf("curl GET %s: code=%d err=%v stdout=%s", path, code, err, stdout)
	}
	return parseCurlOutput(stdout)
}

// parseCurlOutput splits curl -w '\n%{http_code}' output into body and status.
func parseCurlOutput(output string) (int, string, error) {
	output = strings.TrimSpace(output)
	idx := strings.LastIndex(output, "\n")
	if idx < 0 {
		// Single line = just status code, no body
		var code int
		if _, err := fmt.Sscanf(output, "%d", &code); err != nil {
			return 0, "", fmt.Errorf("parse curl status: %q", output)
		}
		return code, "", nil
	}
	body := output[:idx]
	var httpCode int
	if _, err := fmt.Sscanf(strings.TrimSpace(output[idx+1:]), "%d", &httpCode); err != nil {
		return 0, "", fmt.Errorf("parse curl status from %q", output[idx+1:])
	}
	return httpCode, body, nil
}

// Assign sends POST /assign to inject a role/epoch assignment.
func (h *HATarget) Assign(ctx context.Context, epoch uint64, role uint32, leaseTTLMs uint32) error {
	code, body, err := h.curlPost(ctx, "/assign", map[string]interface{}{
		"epoch":        epoch,
		"role":         role,
		"lease_ttl_ms": leaseTTLMs,
	})
	if err != nil {
		return fmt.Errorf("assign request: %w", err)
	}
	if code != http.StatusOK {
		return fmt.Errorf("assign failed (HTTP %d): %s", code, body)
	}
	return nil
}

// AssignRaw sends POST /assign and returns HTTP status code and body.
func (h *HATarget) AssignRaw(ctx context.Context, body interface{}) (int, string, error) {
	return h.curlPost(ctx, "/assign", body)
}

// Status sends GET /status and returns the parsed response.
func (h *HATarget) Status(ctx context.Context) (*StatusResp, error) {
	code, body, err := h.curlGet(ctx, "/status")
	if err != nil {
		return nil, fmt.Errorf("status request: %w", err)
	}
	if code != http.StatusOK {
		return nil, fmt.Errorf("status failed (HTTP %d): %s", code, body)
	}
	var st StatusResp
	if err := json.NewDecoder(strings.NewReader(body)).Decode(&st); err != nil {
		return nil, fmt.Errorf("decode status: %w", err)
	}
	return &st, nil
}

// SetReplica sends POST /replica to configure WAL shipping target.
func (h *HATarget) SetReplica(ctx context.Context, dataAddr, ctrlAddr string) error {
	code, body, err := h.curlPost(ctx, "/replica", map[string]string{
		"data_addr": dataAddr,
		"ctrl_addr": ctrlAddr,
	})
	if err != nil {
		return fmt.Errorf("replica request: %w", err)
	}
	if code != http.StatusOK {
		return fmt.Errorf("replica failed (HTTP %d): %s", code, body)
	}
	return nil
}

// SetReplicaRaw sends POST /replica and returns HTTP status + body.
func (h *HATarget) SetReplicaRaw(ctx context.Context, body interface{}) (int, string, error) {
	return h.curlPost(ctx, "/replica", body)
}

// StartRebuildEndpoint sends POST /rebuild {action:"start"}.
func (h *HATarget) StartRebuildEndpoint(ctx context.Context, listenAddr string) error {
	code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{
		"action":      "start",
		"listen_addr": listenAddr,
	})
	if err != nil {
		return fmt.Errorf("rebuild start: %w", err)
	}
	if code != http.StatusOK {
		return fmt.Errorf("rebuild start failed (HTTP %d): %s", code, body)
	}
	return nil
}

// StartRebuildClient sends POST /rebuild {action:"connect"} to start the
// rebuild client. The client connects to the primary's rebuild server,
// streams WAL/extent data, and transitions from RoleRebuilding to RoleReplica.
// This is non-blocking on the target side; poll WaitForRole("replica") to
// check completion.
func (h *HATarget) StartRebuildClient(ctx context.Context, rebuildAddr string, epoch uint64) error {
	code, body, err := h.curlPost(ctx, "/rebuild", map[string]interface{}{
		"action":       "connect",
		"rebuild_addr": rebuildAddr,
		"epoch":        epoch,
	})
	if err != nil {
		return fmt.Errorf("rebuild connect: %w", err)
	}
	if code != http.StatusOK {
		return fmt.Errorf("rebuild connect failed (HTTP %d): %s", code, body)
	}
	return nil
}

// StopRebuildEndpoint sends POST /rebuild {action:"stop"}.
func (h *HATarget) StopRebuildEndpoint(ctx context.Context) error {
	code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{"action": "stop"})
	if err != nil {
		return fmt.Errorf("rebuild stop: %w", err)
	}
	if code != http.StatusOK {
		return fmt.Errorf("rebuild stop failed (HTTP %d): %s", code, body)
	}
	return nil
}

// WaitForRole polls GET /status until the target reports the expected role.
func (h *HATarget) WaitForRole(ctx context.Context, expectedRole string) error {
	for {
		select {
		case <-ctx.Done():
			return fmt.Errorf("wait for role %s: %w", expectedRole, ctx.Err())
		default:
		}
		st, err := h.Status(ctx)
		if err == nil && st.Role == expectedRole {
			return nil
		}
		time.Sleep(500 * time.Millisecond)
	}
}

// WaitForLSN polls GET /status until wal_head_lsn >= minLSN.
func (h *HATarget) WaitForLSN(ctx context.Context, minLSN uint64) error {
	for {
		select {
		case <-ctx.Done():
			return fmt.Errorf("wait for LSN >= %d: %w", minLSN, ctx.Err())
		default:
		}
		st, err := h.Status(ctx)
		if err == nil && st.WALHeadLSN >= minLSN {
			return nil
		}
		time.Sleep(500 * time.Millisecond)
	}
}