seaweedfs/weed/storage/blockvol/testrunner/actions/bench.go

package actions

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"

	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
)

// RegisterBenchActions registers benchmark-related actions.
func RegisterBenchActions(r *tr.Registry) {
	r.RegisterFunc("fio_json", tr.TierBlock, fioJSON)
	r.RegisterFunc("fio_parse", tr.TierCore, fioParse)
	r.RegisterFunc("bench_compare", tr.TierCore, benchCompare)
	r.RegisterFunc("bench_stats", tr.TierCore, benchStats)
	registerBenchmarkValidation(r)
}

// fioJSON runs fio with JSON output. Supports numjobs for multi-queue testing.
// Params:
//   - device (required): block device path
//   - rw: IO pattern (default: "randwrite")
//   - bs: block size (default: "4k")
//   - iodepth: queue depth per job (default: "32")
//   - numjobs: number of parallel jobs (default: "1")
//   - runtime: seconds (default: "60")
//   - size: file/device size (default: "256M")
//   - name: job name (default: "bench")
//   - rwmixread: read percentage for randrw (optional)
//
// Returns: value = fio JSON output string
func fioJSON(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
	device := act.Params["device"]
	if device == "" {
		return nil, fmt.Errorf("fio_json: device param required")
	}

	rw := paramDefault(act.Params, "rw", "randwrite")
	bs := paramDefault(act.Params, "bs", "4k")
	iodepth := paramDefault(act.Params, "iodepth", "32")
	numjobs := paramDefault(act.Params, "numjobs", "1")
	runtime := paramDefault(act.Params, "runtime", "60")
	size := paramDefault(act.Params, "size", "256M")
	name := paramDefault(act.Params, "name", "bench")

	node, err := GetNode(actx, act.Node)
	if err != nil {
		return nil, err
	}

	cmd := fmt.Sprintf("fio --name=%s --filename=%s --rw=%s --bs=%s --iodepth=%s --numjobs=%s --direct=1 --ioengine=libaio --runtime=%s --time_based --size=%s --group_reporting --output-format=json",
		name, device, rw, bs, iodepth, numjobs, runtime, size)

	if rwmixread := act.Params["rwmixread"]; rwmixread != "" {
		cmd += fmt.Sprintf(" --rwmixread=%s", rwmixread)
	}

	actx.Log("  fio %s bs=%s j=%s qd=%s %ss on %s", rw, bs, numjobs, iodepth, runtime, device)
	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
	if err != nil || code != 0 {
		return nil, fmt.Errorf("fio_json: code=%d stderr=%s err=%v", code, stderr, err)
	}

	return map[string]string{"value": stdout}, nil
}

// fioParse extracts a specific metric from fio JSON output.
// Params:
//   - json_var: name of var containing fio JSON (required)
//   - metric: one of "iops", "bw_bytes", "lat_mean_us", "lat_p50_us", "lat_p99_us", "lat_p999_us" (required)
//   - direction: "read" or "write" (default: auto-detect from rw type)
//
// Returns: value = numeric string
func fioParse(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
	varName := act.Params["json_var"]
	if varName == "" {
		return nil, fmt.Errorf("fio_parse: json_var param required")
	}
	metric := act.Params["metric"]
	if metric == "" {
		return nil, fmt.Errorf("fio_parse: metric param required")
	}

	jsonStr := actx.Vars[varName]
	if jsonStr == "" {
		return nil, fmt.Errorf("fio_parse: var %q is empty", varName)
	}

	val, err := ParseFioMetric(jsonStr, metric, act.Params["direction"])
	if err != nil {
		return nil, fmt.Errorf("fio_parse: %w", err)
	}

	return map[string]string{"value": strconv.FormatFloat(val, 'f', 2, 64)}, nil
}

// benchCompare compares two fio results and asserts a performance gate.
// Params:
//   - a_var: var name for baseline (e.g. iSCSI) fio JSON (required)
//   - b_var: var name for candidate (e.g. NVMe) fio JSON (required)
//   - metric: metric to compare (required, same as fio_parse)
//   - gate: minimum ratio b/a (default: "1.0" = candidate >= baseline)
//   - warn_gate: soft threshold — ratio < gate but >= warn_gate returns success
//     with value prefixed "WARN:" instead of hard-failing (optional)
//   - direction: "read" or "write" (default: auto-detect)
//
// Returns: value = "delta_pct" (e.g. "+14.1%"), prefixed "WARN:" if in warn band.
// Fails only if candidate/baseline < warn_gate (or < gate when warn_gate is unset).
func benchCompare(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
	aVar := act.Params["a_var"]
	bVar := act.Params["b_var"]
	metric := act.Params["metric"]
	if aVar == "" || bVar == "" || metric == "" {
		return nil, fmt.Errorf("bench_compare: a_var, b_var, metric params required")
	}

	gateStr := paramDefault(act.Params, "gate", "1.0")
	gate, err := strconv.ParseFloat(gateStr, 64)
	if err != nil {
		return nil, fmt.Errorf("bench_compare: invalid gate %q: %w", gateStr, err)
	}

	// warn_gate: soft threshold below gate. If ratio is between warn_gate and gate,
	// we return success with a "WARN:" prefix instead of hard-failing.
	warnGate := 0.0
	hasWarnGate := false
	if wg := act.Params["warn_gate"]; wg != "" {
		warnGate, err = strconv.ParseFloat(wg, 64)
		if err != nil {
			return nil, fmt.Errorf("bench_compare: invalid warn_gate %q: %w", wg, err)
		}
		hasWarnGate = true
	}

	direction := act.Params["direction"]

	aJSON := actx.Vars[aVar]
	bJSON := actx.Vars[bVar]
	if aJSON == "" {
		return nil, fmt.Errorf("bench_compare: var %q is empty", aVar)
	}
	if bJSON == "" {
		return nil, fmt.Errorf("bench_compare: var %q is empty", bVar)
	}

	aVal, err := ParseFioMetric(aJSON, metric, direction)
	if err != nil {
		return nil, fmt.Errorf("bench_compare baseline (%s): %w", aVar, err)
	}
	bVal, err := ParseFioMetric(bJSON, metric, direction)
	if err != nil {
		return nil, fmt.Errorf("bench_compare candidate (%s): %w", bVar, err)
	}

	// For latency metrics, lower is better — invert the comparison.
	isLatency := strings.HasPrefix(metric, "lat_")
	var ratio float64
	var deltaStr string

	if aVal == 0 {
		return nil, fmt.Errorf("bench_compare: baseline %s = 0, cannot compute ratio", metric)
	}

	if isLatency {
		// For latency: ratio = baseline/candidate (higher is better = candidate has lower latency)
		ratio = aVal / bVal
		deltaPct := (aVal - bVal) / aVal * 100
		if deltaPct >= 0 {
			deltaStr = fmt.Sprintf("-%.1f%%", deltaPct) // latency decreased = good
		} else {
			deltaStr = fmt.Sprintf("+%.1f%%", -deltaPct) // latency increased = bad
		}
	} else {
		// For throughput: ratio = candidate/baseline (higher is better)
		ratio = bVal / aVal
		deltaPct := (bVal - aVal) / aVal * 100
		if deltaPct >= 0 {
			deltaStr = fmt.Sprintf("+%.1f%%", deltaPct)
		} else {
			deltaStr = fmt.Sprintf("%.1f%%", deltaPct)
		}
	}

	actx.Log("  %s: baseline=%.1f candidate=%.1f delta=%s ratio=%.3f gate=%.2f",
		metric, aVal, bVal, deltaStr, ratio, gate)

	if ratio < gate {
		// If warn_gate is set and ratio >= warn_gate, return success with WARN prefix.
		if hasWarnGate && ratio >= warnGate {
			actx.Log("  WARN: ratio %.3f below gate %.2f but above warn_gate %.2f", ratio, gate, warnGate)
			return map[string]string{"value": "WARN:" + deltaStr}, nil
		}
		return nil, fmt.Errorf("bench_compare FAIL: %s ratio=%.3f < gate=%.2f (baseline=%.1f candidate=%.1f delta=%s)",
			metric, ratio, gate, aVal, bVal, deltaStr)
	}

	return map[string]string{"value": deltaStr}, nil
}

// --- fio JSON parsing ---

// fioOutput represents the top-level fio JSON output.
type fioOutput struct {
	Jobs []fioJob `json:"jobs"`
}

type fioJob struct {
	JobName string      `json:"jobname"`
	Read    fioJobStats `json:"read"`
	Write   fioJobStats `json:"write"`
}

type fioJobStats struct {
	IOPS    float64    `json:"iops"`
	BWBytes float64    `json:"bw_bytes"`
	LatNS   fioLatency `json:"lat_ns"`
	CLatNS  fioLatency `json:"clat_ns"`
}

type fioLatency struct {
	Mean       float64            `json:"mean"`
	Percentile map[string]float64 `json:"percentile"`
}

// ParseFioMetric extracts a named metric from fio JSON or returns a plain
// numeric value directly. This allows bench_compare to accept either raw fio
// JSON output or pre-aggregated scalar values (e.g. from bench_stats or
// phase repeat/aggregate).
//
// direction: "read", "write", or "" (auto-detect: use whichever has IOPS > 0).
// Supported metrics: "iops", "bw_bytes", "bw_mb", "lat_mean_us", "lat_p50_us", "lat_p99_us", "lat_p999_us"
func ParseFioMetric(input, metric, direction string) (float64, error) {
	// Try plain numeric value first (from aggregation or bench_stats).
	trimmed := strings.TrimSpace(input)
	if v, err := strconv.ParseFloat(trimmed, 64); err == nil {
		return v, nil
	}

	// Try quoted numeric string (e.g. "15322.00").
	if len(trimmed) >= 2 && trimmed[0] == '"' && trimmed[len(trimmed)-1] == '"' {
		if v, err := strconv.ParseFloat(trimmed[1:len(trimmed)-1], 64); err == nil {
			return v, nil
		}
	}

	// Parse as fio JSON.
	var output fioOutput
	if err := json.Unmarshal([]byte(input), &output); err != nil {
		return 0, fmt.Errorf("parse fio metric: input is neither a number nor valid fio JSON: %w", err)
	}
	if len(output.Jobs) == 0 {
		return 0, fmt.Errorf("fio JSON has no jobs")
	}

	// Use first job (group_reporting merges into one).
	job := output.Jobs[0]

	// Auto-detect direction.
	var stats fioJobStats
	switch direction {
	case "read":
		stats = job.Read
	case "write":
		stats = job.Write
	default:
		if job.Write.IOPS > 0 {
			stats = job.Write
		} else {
			stats = job.Read
		}
	}

	switch metric {
	case "iops":
		return stats.IOPS, nil
	case "bw_bytes":
		return stats.BWBytes, nil
	case "bw_mb":
		return stats.BWBytes / (1024 * 1024), nil
	case "lat_mean_us":
		return stats.LatNS.Mean / 1000, nil // ns → µs
	case "lat_p50_us":
		return getPercentileWithFallback(stats, "50.000000") / 1000, nil
	case "lat_p99_us":
		return getPercentileWithFallback(stats, "99.000000") / 1000, nil
	case "lat_p999_us":
		return getPercentileWithFallback(stats, "99.900000") / 1000, nil
	default:
		return 0, fmt.Errorf("unknown metric %q", metric)
	}
}

func getPercentile(lat fioLatency, key string) float64 {
	if lat.Percentile == nil {
		return 0
	}
	return lat.Percentile[key]
}

// getPercentileWithFallback tries clat_ns first (fio puts percentiles there),
// then falls back to lat_ns.
func getPercentileWithFallback(stats fioJobStats, key string) float64 {
	if v := getPercentile(stats.CLatNS, key); v != 0 {
		return v
	}
	return getPercentile(stats.LatNS, key)
}

// benchStats computes statistics from a comma-separated list of values.
// Useful for aggregating results from multiple runs outside the phase repeat system.
// Params:
//   - values_var: name of var containing comma-separated numeric values (required)
//   - trim_pct: percentage of outliers to trim from each end (default: "20")
//   - label: label for log output (default: "bench_stats")
//
// Returns: value = median. Also sets {save_as}_mean, _stddev, _min, _max, _n.
func benchStats(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
	varName := act.Params["values_var"]
	if varName == "" {
		return nil, fmt.Errorf("bench_stats: values_var param required")
	}
	valStr := actx.Vars[varName]
	if valStr == "" {
		return nil, fmt.Errorf("bench_stats: var %q is empty", varName)
	}

	trimPct := 20
	if tp := act.Params["trim_pct"]; tp != "" {
		if v, err := strconv.Atoi(tp); err == nil {
			trimPct = v
		}
	}
	label := act.Params["label"]
	if label == "" {
		label = "bench_stats"
	}

	// Parse comma-separated values.
	parts := strings.Split(valStr, ",")
	var values []float64
	for _, p := range parts {
		p = strings.TrimSpace(p)
		if p == "" {
			continue
		}
		f, err := strconv.ParseFloat(p, 64)
		if err != nil {
			return nil, fmt.Errorf("bench_stats: invalid value %q in %s: %w", p, varName, err)
		}
		values = append(values, f)
	}
	if len(values) == 0 {
		return nil, fmt.Errorf("bench_stats: no numeric values in %s", varName)
	}

	// Trim outliers and compute stats.
	trimmed := trimValues(values, trimPct)
	stats := tr.ComputeStats(trimmed)

	actx.Log("  [%s] n=%d median=%.2f mean=%.2f stddev=%.2f min=%.2f max=%.2f (trimmed %d%% from %d)",
		label, stats.Count, stats.P50, stats.Mean, stats.StdDev, stats.Min, stats.Max, trimPct, len(values))

	result := map[string]string{
		"value": strconv.FormatFloat(stats.P50, 'f', 2, 64),
	}

	// Store detailed stats as __-prefixed vars for auto-propagation.
	if act.SaveAs != "" {
		actx.Vars[act.SaveAs+"_mean"] = strconv.FormatFloat(stats.Mean, 'f', 2, 64)
		actx.Vars[act.SaveAs+"_stddev"] = strconv.FormatFloat(stats.StdDev, 'f', 2, 64)
		actx.Vars[act.SaveAs+"_min"] = strconv.FormatFloat(stats.Min, 'f', 2, 64)
		actx.Vars[act.SaveAs+"_max"] = strconv.FormatFloat(stats.Max, 'f', 2, 64)
		actx.Vars[act.SaveAs+"_n"] = strconv.Itoa(stats.Count)
	}

	return result, nil
}

// trimValues removes the top and bottom pct% of values.
func trimValues(values []float64, pct int) []float64 {
	if len(values) <= 2 || pct <= 0 {
		return values
	}
	sorted := make([]float64, len(values))
	copy(sorted, values)
	sort.Float64s(sorted)

	trim := int(math.Round(float64(len(sorted)) * float64(pct) / 100.0))
	if trim*2 >= len(sorted) {
		trim = (len(sorted) - 1) / 2
	}
	return sorted[trim : len(sorted)-trim]
}

func paramDefault(params map[string]string, key, def string) string {
	if v := params[key]; v != "" {
		return v
	}
	return def
}

// FormatBenchReport generates a human-readable A/B comparison table.
// results is a list of {workload, metric, baselineVal, candidateVal, deltaPct, gate, pass}.
func FormatBenchReport(results []BenchResult) string {
	var b strings.Builder
	b.WriteString(fmt.Sprintf("%-24s | %12s | %12s | %8s | %s\n", "Workload", "Baseline", "Candidate", "Delta", "Gate"))
	b.WriteString(strings.Repeat("-", 76) + "\n")
	for _, r := range results {
		status := "PASS"
		if !r.Pass {
			status = "FAIL"
			if r.Ratio >= 0.9 {
				status = "WARN"
			}
		}
		b.WriteString(fmt.Sprintf("%-24s | %12.1f | %12.1f | %7s | %s\n",
			r.Workload, r.Baseline, r.Candidate, r.Delta, status))
	}
	return b.String()
}

// BenchResult holds one row of A/B comparison.
type BenchResult struct {
	Workload  string
	Metric    string
	Baseline  float64
	Candidate float64
	Delta     string
	Ratio     float64
	Gate      float64
	Pass      bool
}

// ComputeBenchResult computes a single A/B comparison row.
func ComputeBenchResult(workload, metric string, baseline, candidate, gate float64) BenchResult {
	isLatency := strings.HasPrefix(metric, "lat_")
	var ratio float64
	var delta string

	if baseline == 0 {
		return BenchResult{Workload: workload, Metric: metric, Pass: false, Delta: "N/A"}
	}

	if isLatency {
		ratio = baseline / candidate
		deltaPct := (baseline - candidate) / baseline * 100
		if deltaPct >= 0 {
			delta = fmt.Sprintf("-%.1f%%", deltaPct)
		} else {
			delta = fmt.Sprintf("+%.1f%%", math.Abs(deltaPct))
		}
	} else {
		ratio = candidate / baseline
		deltaPct := (candidate - baseline) / baseline * 100
		if deltaPct >= 0 {
			delta = fmt.Sprintf("+%.1f%%", deltaPct)
		} else {
			delta = fmt.Sprintf("%.1f%%", deltaPct)
		}
	}

	return BenchResult{
		Workload:  workload,
		Metric:    metric,
		Baseline:  baseline,
		Candidate: candidate,
		Delta:     delta,
		Ratio:     ratio,
		Gate:      gate,
		Pass:      ratio >= gate,
	}
}