mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-17 01:05:29 +01:00
Phase 8 closes with all 6 checkpoints done (CP8-1 through CP8-5 + CP8-3-1): - CP8-5: 12/12 enterprise QA scenarios PASS on real hardware (m01/M02) - Master-authoritative lease grants (BUG-CP85-11): master renews primary write leases on every heartbeat response, replacing retain-until-confirmed assignment queue semantics that caused 30s lease expiry - Post-rebuild WAL shipping gap fix (BUG-CP85-1): syncLSNAfterRebuild advances replica nextLSN so WAL entries are accepted after rebuild - Block heartbeat startup race fix (BUG-CP85-10): dynamic blockService check on each tick instead of one-shot at loop start - 8 new tests: 4 engine lease grant + 4 registry lease grant - 13 new YAML scenarios: chaos (kill-loop, partition, disk-full), database integrity (sqlite crash, ext4 fsck), perf baseline, metrics verify, snapshot stress, expand-failover, session storm, role flap, 24h soak - 12 new testrunner actions (database, fsck, grep_log, write_loop_bg, stop_bg, assert_metric_gt/eq/lt) + phase repeat support - Docker compose setup + getting-started guide for block storage users - 960+ cumulative unit tests, 24 YAML scenarios Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
321 lines
8.9 KiB
Go
321 lines
8.9 KiB
Go
package actions
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
|
|
)
|
|
|
|
// RegisterIOActions registers IO-related actions.
|
|
func RegisterIOActions(r *tr.Registry) {
|
|
r.RegisterFunc("dd_write", tr.TierBlock, ddWrite)
|
|
r.RegisterFunc("dd_read_md5", tr.TierBlock, ddReadMD5)
|
|
r.RegisterFunc("fio", tr.TierBlock, fioAction)
|
|
r.RegisterFunc("fio_verify", tr.TierBlock, fioVerify)
|
|
r.RegisterFunc("mkfs", tr.TierBlock, mkfsAction)
|
|
r.RegisterFunc("mount", tr.TierBlock, mountAction)
|
|
r.RegisterFunc("umount", tr.TierBlock, umountAction)
|
|
r.RegisterFunc("write_loop_bg", tr.TierBlock, writeLoopBg)
|
|
r.RegisterFunc("stop_bg", tr.TierBlock, stopBg)
|
|
}
|
|
|
|
// ddWrite writes random data using dd, returns the md5 checksum.
|
|
func ddWrite(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("dd_write: device param required")
|
|
}
|
|
bs := act.Params["bs"]
|
|
if bs == "" {
|
|
bs = "1M"
|
|
}
|
|
count := act.Params["count"]
|
|
if count == "" {
|
|
count = "1"
|
|
}
|
|
oflag := act.Params["oflag"]
|
|
if oflag == "" {
|
|
oflag = "direct"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Generate random data to temp file, write to device, compute md5.
|
|
tmpFile := "/tmp/sw-test-runner-dd-data"
|
|
genCmd := fmt.Sprintf("dd if=/dev/urandom of=%s bs=%s count=%s 2>/dev/null", tmpFile, bs, count)
|
|
_, stderr, code, err := node.RunRoot(ctx, genCmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("dd_write gen: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
writeCmd := fmt.Sprintf("dd if=%s of=%s bs=%s oflag=%s conv=fsync", tmpFile, device, bs, oflag)
|
|
if seek := act.Params["seek"]; seek != "" {
|
|
writeCmd += fmt.Sprintf(" seek=%s", seek)
|
|
}
|
|
writeCmd += " 2>/dev/null"
|
|
_, stderr, code, err = node.RunRoot(ctx, writeCmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("dd_write: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
md5Cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
|
|
stdout, stderr, code, err := node.RunRoot(ctx, md5Cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("dd_write md5: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
|
|
|
|
md5 := strings.TrimSpace(stdout)
|
|
if md5 == "" {
|
|
return nil, fmt.Errorf("dd_write: empty md5")
|
|
}
|
|
|
|
return map[string]string{"value": md5}, nil
|
|
}
|
|
|
|
// ddReadMD5 reads from device using dd and returns md5.
|
|
func ddReadMD5(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("dd_read_md5: device param required")
|
|
}
|
|
bs := act.Params["bs"]
|
|
if bs == "" {
|
|
bs = "1M"
|
|
}
|
|
count := act.Params["count"]
|
|
if count == "" {
|
|
count = "1"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tmpFile := "/tmp/sw-test-runner-dd-read"
|
|
readCmd := fmt.Sprintf("dd if=%s of=%s bs=%s count=%s iflag=direct", device, tmpFile, bs, count)
|
|
if skip := act.Params["skip"]; skip != "" {
|
|
readCmd += fmt.Sprintf(" skip=%s", skip)
|
|
}
|
|
readCmd += " 2>/dev/null"
|
|
_, stderr, code, err := node.RunRoot(ctx, readCmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("dd_read_md5 read: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
|
|
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("dd_read_md5: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
|
|
|
|
md5 := strings.TrimSpace(stdout)
|
|
if md5 == "" {
|
|
return nil, fmt.Errorf("dd_read_md5: empty md5")
|
|
}
|
|
|
|
return map[string]string{"value": md5}, nil
|
|
}
|
|
|
|
func fioAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("fio: device param required")
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rw := act.Params["rw"]
|
|
if rw == "" {
|
|
rw = "randwrite"
|
|
}
|
|
bs := act.Params["bs"]
|
|
if bs == "" {
|
|
bs = "4k"
|
|
}
|
|
iodepth := act.Params["iodepth"]
|
|
if iodepth == "" {
|
|
iodepth = "32"
|
|
}
|
|
runtime := act.Params["runtime"]
|
|
if runtime == "" {
|
|
runtime = "10"
|
|
}
|
|
name := act.Params["name"]
|
|
if name == "" {
|
|
name = "fio_test"
|
|
}
|
|
|
|
cmd := fmt.Sprintf("fio --name=%s --filename=%s --rw=%s --bs=%s --iodepth=%s --direct=1 --runtime=%s --time_based --output-format=json",
|
|
name, device, rw, bs, iodepth, runtime)
|
|
if size := act.Params["size"]; size != "" {
|
|
cmd += fmt.Sprintf(" --size=%s", size)
|
|
}
|
|
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("fio: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
return map[string]string{"value": stdout}, nil
|
|
}
|
|
|
|
func fioVerify(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("fio_verify: device param required")
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
bs := act.Params["bs"]
|
|
if bs == "" {
|
|
bs = "4k"
|
|
}
|
|
size := act.Params["size"]
|
|
if size == "" {
|
|
size = "10M"
|
|
}
|
|
|
|
// Write with verify pattern, then read+verify.
|
|
cmd := fmt.Sprintf("fio --name=verify --filename=%s --rw=write --bs=%s --size=%s --direct=1 --verify=crc32c --do_verify=1 --output-format=json",
|
|
device, bs, size)
|
|
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("fio_verify: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
return map[string]string{"value": stdout}, nil
|
|
}
|
|
|
|
func mkfsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("mkfs: device param required")
|
|
}
|
|
fstype := act.Params["fstype"]
|
|
if fstype == "" {
|
|
fstype = "ext4"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cmd := fmt.Sprintf("mkfs.%s -F %s", fstype, device)
|
|
_, stderr, code, err := node.RunRoot(ctx, cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("mkfs: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func mountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("mount: device param required")
|
|
}
|
|
mountpoint := act.Params["mountpoint"]
|
|
if mountpoint == "" {
|
|
mountpoint = "/mnt/test"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", mountpoint))
|
|
_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("mount %s %s", device, mountpoint))
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("mount: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
return map[string]string{"value": mountpoint}, nil
|
|
}
|
|
|
|
func umountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
mountpoint := act.Params["mountpoint"]
|
|
if mountpoint == "" {
|
|
mountpoint = "/mnt/test"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("umount %s", mountpoint))
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("umount: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// writeLoopBg starts a background dd write loop. Returns PID.
|
|
// Params: device (required), bs (default: "4k"), oflag (default: "direct")
|
|
func writeLoopBg(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
device := act.Params["device"]
|
|
if device == "" {
|
|
return nil, fmt.Errorf("write_loop_bg: device param required")
|
|
}
|
|
bs := act.Params["bs"]
|
|
if bs == "" {
|
|
bs = "4k"
|
|
}
|
|
oflag := act.Params["oflag"]
|
|
if oflag == "" {
|
|
oflag = "direct"
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cmd := fmt.Sprintf("setsid bash -c 'while true; do dd if=/dev/urandom of=%s bs=%s count=1 oflag=%s conv=notrunc 2>/dev/null; done' &>/tmp/sw_bg.log & echo $!",
|
|
device, bs, oflag)
|
|
stdout, stderr, code, err := node.RunRoot(ctx, cmd)
|
|
if err != nil || code != 0 {
|
|
return nil, fmt.Errorf("write_loop_bg: code=%d stderr=%s err=%v", code, stderr, err)
|
|
}
|
|
|
|
pid := strings.TrimSpace(stdout)
|
|
if pid == "" {
|
|
return nil, fmt.Errorf("write_loop_bg: empty PID")
|
|
}
|
|
|
|
return map[string]string{"value": pid}, nil
|
|
}
|
|
|
|
// stopBg kills a background process by PID.
|
|
// Params: pid (required)
|
|
func stopBg(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
|
|
pid := act.Params["pid"]
|
|
if pid == "" {
|
|
return nil, fmt.Errorf("stop_bg: pid param required")
|
|
}
|
|
|
|
node, err := getNode(actx, act.Node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cmd := fmt.Sprintf("kill %s; wait %s 2>/dev/null || true", pid, pid)
|
|
node.RunRoot(ctx, cmd)
|
|
|
|
return nil, nil
|
|
}
|