Files
seaweedfs/weed/storage/blockvol/testrunner/actions/io.go
T
Ping QiuandClaude Opus 4.6 9acd187587 feat: Phase 8 complete -- CP8-5 stability gate, lease grant fix, Docker e2e, 13 chaos scenarios
Phase 8 closes with all 6 checkpoints done (CP8-1 through CP8-5 + CP8-3-1):
- CP8-5: 12/12 enterprise QA scenarios PASS on real hardware (m01/M02)
- Master-authoritative lease grants (BUG-CP85-11): master renews primary
  write leases on every heartbeat response, replacing retain-until-confirmed
  assignment queue semantics that caused 30s lease expiry
- Post-rebuild WAL shipping gap fix (BUG-CP85-1): syncLSNAfterRebuild
  advances replica nextLSN so WAL entries are accepted after rebuild
- Block heartbeat startup race fix (BUG-CP85-10): dynamic blockService
  check on each tick instead of one-shot at loop start
- 8 new tests: 4 engine lease grant + 4 registry lease grant
- 13 new YAML scenarios: chaos (kill-loop, partition, disk-full),
  database integrity (sqlite crash, ext4 fsck), perf baseline,
  metrics verify, snapshot stress, expand-failover, session storm,
  role flap, 24h soak
- 12 new testrunner actions (database, fsck, grep_log, write_loop_bg,
  stop_bg, assert_metric_gt/eq/lt) + phase repeat support
- Docker compose setup + getting-started guide for block storage users
- 960+ cumulative unit tests, 24 YAML scenarios

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 21:30:14 -08:00

321 lines
8.9 KiB
Go

package actions
import (
"context"
"fmt"
"strings"
tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
)
// RegisterIOActions registers IO-related actions.
func RegisterIOActions(r *tr.Registry) {
r.RegisterFunc("dd_write", tr.TierBlock, ddWrite)
r.RegisterFunc("dd_read_md5", tr.TierBlock, ddReadMD5)
r.RegisterFunc("fio", tr.TierBlock, fioAction)
r.RegisterFunc("fio_verify", tr.TierBlock, fioVerify)
r.RegisterFunc("mkfs", tr.TierBlock, mkfsAction)
r.RegisterFunc("mount", tr.TierBlock, mountAction)
r.RegisterFunc("umount", tr.TierBlock, umountAction)
r.RegisterFunc("write_loop_bg", tr.TierBlock, writeLoopBg)
r.RegisterFunc("stop_bg", tr.TierBlock, stopBg)
}
// ddWrite writes random data using dd, returns the md5 checksum.
func ddWrite(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("dd_write: device param required")
}
bs := act.Params["bs"]
if bs == "" {
bs = "1M"
}
count := act.Params["count"]
if count == "" {
count = "1"
}
oflag := act.Params["oflag"]
if oflag == "" {
oflag = "direct"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
// Generate random data to temp file, write to device, compute md5.
tmpFile := "/tmp/sw-test-runner-dd-data"
genCmd := fmt.Sprintf("dd if=/dev/urandom of=%s bs=%s count=%s 2>/dev/null", tmpFile, bs, count)
_, stderr, code, err := node.RunRoot(ctx, genCmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("dd_write gen: code=%d stderr=%s err=%v", code, stderr, err)
}
writeCmd := fmt.Sprintf("dd if=%s of=%s bs=%s oflag=%s conv=fsync", tmpFile, device, bs, oflag)
if seek := act.Params["seek"]; seek != "" {
writeCmd += fmt.Sprintf(" seek=%s", seek)
}
writeCmd += " 2>/dev/null"
_, stderr, code, err = node.RunRoot(ctx, writeCmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("dd_write: code=%d stderr=%s err=%v", code, stderr, err)
}
md5Cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
stdout, stderr, code, err := node.RunRoot(ctx, md5Cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("dd_write md5: code=%d stderr=%s err=%v", code, stderr, err)
}
node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
md5 := strings.TrimSpace(stdout)
if md5 == "" {
return nil, fmt.Errorf("dd_write: empty md5")
}
return map[string]string{"value": md5}, nil
}
// ddReadMD5 reads from device using dd and returns md5.
func ddReadMD5(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("dd_read_md5: device param required")
}
bs := act.Params["bs"]
if bs == "" {
bs = "1M"
}
count := act.Params["count"]
if count == "" {
count = "1"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
tmpFile := "/tmp/sw-test-runner-dd-read"
readCmd := fmt.Sprintf("dd if=%s of=%s bs=%s count=%s iflag=direct", device, tmpFile, bs, count)
if skip := act.Params["skip"]; skip != "" {
readCmd += fmt.Sprintf(" skip=%s", skip)
}
readCmd += " 2>/dev/null"
_, stderr, code, err := node.RunRoot(ctx, readCmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("dd_read_md5 read: code=%d stderr=%s err=%v", code, stderr, err)
}
cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("dd_read_md5: code=%d stderr=%s err=%v", code, stderr, err)
}
node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
md5 := strings.TrimSpace(stdout)
if md5 == "" {
return nil, fmt.Errorf("dd_read_md5: empty md5")
}
return map[string]string{"value": md5}, nil
}
func fioAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("fio: device param required")
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
rw := act.Params["rw"]
if rw == "" {
rw = "randwrite"
}
bs := act.Params["bs"]
if bs == "" {
bs = "4k"
}
iodepth := act.Params["iodepth"]
if iodepth == "" {
iodepth = "32"
}
runtime := act.Params["runtime"]
if runtime == "" {
runtime = "10"
}
name := act.Params["name"]
if name == "" {
name = "fio_test"
}
cmd := fmt.Sprintf("fio --name=%s --filename=%s --rw=%s --bs=%s --iodepth=%s --direct=1 --runtime=%s --time_based --output-format=json",
name, device, rw, bs, iodepth, runtime)
if size := act.Params["size"]; size != "" {
cmd += fmt.Sprintf(" --size=%s", size)
}
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("fio: code=%d stderr=%s err=%v", code, stderr, err)
}
return map[string]string{"value": stdout}, nil
}
func fioVerify(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("fio_verify: device param required")
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
bs := act.Params["bs"]
if bs == "" {
bs = "4k"
}
size := act.Params["size"]
if size == "" {
size = "10M"
}
// Write with verify pattern, then read+verify.
cmd := fmt.Sprintf("fio --name=verify --filename=%s --rw=write --bs=%s --size=%s --direct=1 --verify=crc32c --do_verify=1 --output-format=json",
device, bs, size)
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("fio_verify: code=%d stderr=%s err=%v", code, stderr, err)
}
return map[string]string{"value": stdout}, nil
}
func mkfsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("mkfs: device param required")
}
fstype := act.Params["fstype"]
if fstype == "" {
fstype = "ext4"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
cmd := fmt.Sprintf("mkfs.%s -F %s", fstype, device)
_, stderr, code, err := node.RunRoot(ctx, cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("mkfs: code=%d stderr=%s err=%v", code, stderr, err)
}
return nil, nil
}
func mountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("mount: device param required")
}
mountpoint := act.Params["mountpoint"]
if mountpoint == "" {
mountpoint = "/mnt/test"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", mountpoint))
_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("mount %s %s", device, mountpoint))
if err != nil || code != 0 {
return nil, fmt.Errorf("mount: code=%d stderr=%s err=%v", code, stderr, err)
}
return map[string]string{"value": mountpoint}, nil
}
func umountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
mountpoint := act.Params["mountpoint"]
if mountpoint == "" {
mountpoint = "/mnt/test"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("umount %s", mountpoint))
if err != nil || code != 0 {
return nil, fmt.Errorf("umount: code=%d stderr=%s err=%v", code, stderr, err)
}
return nil, nil
}
// writeLoopBg starts a background dd write loop. Returns PID.
// Params: device (required), bs (default: "4k"), oflag (default: "direct")
func writeLoopBg(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
device := act.Params["device"]
if device == "" {
return nil, fmt.Errorf("write_loop_bg: device param required")
}
bs := act.Params["bs"]
if bs == "" {
bs = "4k"
}
oflag := act.Params["oflag"]
if oflag == "" {
oflag = "direct"
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
cmd := fmt.Sprintf("setsid bash -c 'while true; do dd if=/dev/urandom of=%s bs=%s count=1 oflag=%s conv=notrunc 2>/dev/null; done' &>/tmp/sw_bg.log & echo $!",
device, bs, oflag)
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
if err != nil || code != 0 {
return nil, fmt.Errorf("write_loop_bg: code=%d stderr=%s err=%v", code, stderr, err)
}
pid := strings.TrimSpace(stdout)
if pid == "" {
return nil, fmt.Errorf("write_loop_bg: empty PID")
}
return map[string]string{"value": pid}, nil
}
// stopBg kills a background process by PID.
// Params: pid (required)
func stopBg(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
pid := act.Params["pid"]
if pid == "" {
return nil, fmt.Errorf("stop_bg: pid param required")
}
node, err := getNode(actx, act.Node)
if err != nil {
return nil, err
}
cmd := fmt.Sprintf("kill %s; wait %s 2>/dev/null || true", pid, pid)
node.RunRoot(ctx, cmd)
return nil, nil
}