mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-14 18:18:35 +02:00
test: add FUSE database load/durability/perf benchmark (#9980)
* test: add FUSE database load/durability/perf benchmark Runs MySQL (InnoDB) and SQLite with ~1GB datadirs on a SeaweedFS FUSE mount. Two parts: - durability: normal shutdown, kill -9, and crash-during-write all keep every fsync-committed row (verified by integrity check + row count + contiguous prefix + per-row CRC). - performance: FUSE vs the same local disk. fsync/commit latency is the dominant cost (~0.13ms -> ~1.18ms), so small transactions run ~9-12x slower while bulk loads and warm reads stay close. Harness is path-independent (runtime under $SEAWEED_BENCH_WORK) and only touches its own processes on non-default ports. * test/benchmark/fuse_db: portable to Linux + crash-safe progress - export MYSQL_BIN; mysql_bench.py falls back to PATH when unset - unmount via fusermount/fusermount3 (non-root Linux), then umount/diskutil - atomic progress write (tmp+fsync+rename); treat empty progress file as 0 - reuse a single PRNG in the perf probe so RNG init doesn't skew timings * test/benchmark/fuse_db: validate inputs, add subprocess timeout - mysql_bench.py: 1800s timeout on mysql CLI calls; reject db names that aren't plain identifiers (interpolated into SQL) - sqlite_gen.py / sqlite_verify.py: allowlist journal/synchronous modes and the verify mode so a typo can't silently weaken durability or relax checks - run_mysql.sh: durable atomic progress write (tmp+fsync+rename), matching sqlite_gen.py; quote $LB in both crash-test verify calls
This commit is contained in:
1 parent
9266aaa88e
commit
76783f3d71
10 files changed
+827
No files matched your search
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
# Deterministic ~1GB SQLite load on the FUSE mount.
|
||||
# Each row: id, payload (4096 deterministic bytes derived from id), chk=crc32(payload).
|
||||
# Commits in batches; after each commit, writes the committed row count to a
|
||||
# progress file on LOCAL disk and fsyncs it -> a durable record of what was
|
||||
# committed, used to verify "no committed data lost" after a crash.
|
||||
import sys, os, sqlite3, zlib, random
|
||||
|
||||
db_path = sys.argv[1]
|
||||
total_rows = int(sys.argv[2])
|
||||
batch = int(sys.argv[3])
|
||||
progress_path= sys.argv[4] # LOCAL disk
|
||||
ref_path = sys.argv[5] if len(sys.argv) > 5 else None
|
||||
sync_mode = sys.argv[6] if len(sys.argv) > 6 else "FULL"
|
||||
journal_mode = sys.argv[7] if len(sys.argv) > 7 else "DELETE"
|
||||
|
||||
# Validate before interpolating into PRAGMAs: a typo must not silently weaken
|
||||
# durability and invalidate the comparison / crash-survival claims.
|
||||
if sync_mode.upper() not in {"OFF", "NORMAL", "FULL", "EXTRA"}:
|
||||
raise SystemExit(f"invalid synchronous mode: {sync_mode!r}")
|
||||
if journal_mode.upper() not in {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}:
|
||||
raise SystemExit(f"invalid journal mode: {journal_mode!r}")
|
||||
|
||||
def payload(i: int) -> bytes:
|
||||
# deterministic but incompressible (PRNG seeded by id) -> 4096 bytes that
|
||||
# actually consume ~1GB on the volume servers (defeats gzip storage).
|
||||
return random.Random(i).randbytes(4096)
|
||||
|
||||
con = sqlite3.connect(db_path, isolation_level=None)
|
||||
con.execute(f"PRAGMA journal_mode={journal_mode};")
|
||||
con.execute(f"PRAGMA synchronous={sync_mode};")
|
||||
con.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);")
|
||||
|
||||
def write_progress(n):
|
||||
# atomic: write tmp + fsync + rename, so a kill -9 mid-write can't leave an
|
||||
# empty/torn progress file (the crash-test lower bound must stay trustworthy).
|
||||
tmp = progress_path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
f.write(str(n)); f.flush(); os.fsync(f.fileno())
|
||||
os.replace(tmp, progress_path)
|
||||
|
||||
done = 0
|
||||
sum_chk = 0
|
||||
sum_id = 0
|
||||
while done < total_rows:
|
||||
end = min(done + batch, total_rows)
|
||||
rows = []
|
||||
for i in range(done, end):
|
||||
p = payload(i); c = zlib.crc32(p)
|
||||
rows.append((i, p, c)); sum_chk += c; sum_id += i
|
||||
con.execute("BEGIN;")
|
||||
con.executemany("INSERT INTO t(id,payload,chk) VALUES(?,?,?);", rows)
|
||||
con.execute("COMMIT;")
|
||||
done = end
|
||||
write_progress(done)
|
||||
print(f"committed {done}/{total_rows}", flush=True)
|
||||
|
||||
if ref_path:
|
||||
with open(ref_path, "w") as f:
|
||||
f.write(f"{total_rows} {sum_chk} {sum_id}\n"); f.flush(); os.fsync(f.fileno())
|
||||
con.close()
|
||||
print("DONE", flush=True)
|
||||
Reference in new issue
Block a user