Files
seaweedfs/test/fuse_dlm/framework_test.go
T
c76fb36416 test(fuse): two mounts staging the same files without -dlm (#12704)
* test(fuse): two mounts staging the same files without -dlm

Two jobs that extract the same archive into one shared directory, each
through its own mount and without -dlm, both create many of the same
files: a file that exists is skipped, otherwise it is created, written
and closed, then chmod'ed and given its mtime. Before #12593, #12594 and
#12596 the attribute update and the content flush sent the mount's whole
cached entry. Sent after the other mount had rewritten the file, the
update put back chunks that the rewrite had dropped and the filer had
deleted, so the entry pointed at deleted needles and the file failed to
read once its volume was vacuumed. Nothing told the writers.

TestStageSameFilesWithoutDLM runs that on the fuse_dlm cluster with the
mounts started without -dlm (startTestCluster takes a dlm flag;
startDLMTestCluster is unchanged) and checks every staged file: each
chunk is still on the volume server (HEAD on the fid, without
readDeleted) and the filer serves the bytes written. What the losing
writer is told depends on the version and is only logged.

With a 4.53 weed binary the test fails: 66, 71 and 70 of the 400 files
were broken in three runs. With master it passes in about 48 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(fuse): bound the staging test's filer and volume checks

The checks used the default HTTP client and a background context, so a
server that accepts a request and stops answering held the test until the
package timeout instead of failing the check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(fuse): fail the staging test unless both mounts wrote the same files

A mount whose MkdirAll failed returned early, and a mount that started
late could skip every file, so the test passed with one writer. Start both
loops together, fail on setup errors, and require files both mounts wrote.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(fuse): bound the staging test's checks with one deadline

A per-call timeout still let 400 hung lookups and reads run for hours.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2026-11-17 17:08:39 -08:00

574 lines
17 KiB
Go

//go:build linux || darwin
package fuse_dlm
import (
"context"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const filerGroup = "fuse-dlm-test"
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
// mounts for testing DLM-based cross-mount write coordination, and what two
// mounts do without it.
type dlmTestCluster struct {
t testing.TB
baseDir string
weedBinary string
dlm bool // the mounts run with -dlm
masterPort int
masterGrpcPort int
volumePort int
volumeGrpcPort int
filerPorts [2]int
filerGrpcPorts [2]int
mountPoints [2]string
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
filerCmds [2]*exec.Cmd
mountCmds [2]*exec.Cmd
logFiles []*os.File
cleanupOnce sync.Once
}
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
return startTestCluster(t, true)
}
// startTestCluster starts the cluster; dlm says whether the mounts run with -dlm.
func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
binary := findWeedBinary()
if binary == "" {
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
}
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
require.NoError(t, err)
c := &dlmTestCluster{
t: t,
baseDir: baseDir,
weedBinary: binary,
dlm: dlm,
}
// Register cleanup early so processes are stopped even if a require fails below.
t.Cleanup(c.Stop)
// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
ports := allocatePorts(t, 8)
c.masterPort = ports[0]
c.masterGrpcPort = ports[1]
c.volumePort = ports[2]
c.volumeGrpcPort = ports[3]
c.filerPorts[0] = ports[4]
c.filerGrpcPorts[0] = ports[5]
c.filerPorts[1] = ports[6]
c.filerGrpcPorts[1] = ports[7]
// Write empty security.toml
configDir := filepath.Join(baseDir, "config")
require.NoError(t, os.MkdirAll(configDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
// Start master
require.NoError(t, c.startMaster(configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
"master not ready\n%s", c.tailLog("master"))
// Start volume
require.NoError(t, c.startVolume(configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
"volume not ready\n%s", c.tailLog("volume"))
require.NoError(t, c.waitForVolumeRegistered(30*time.Second),
"volume server registration\n%s", c.tailLog("master"))
// Start 2 filers
for i := 0; i < 2; i++ {
require.NoError(t, c.startFiler(i, configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
}
require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
// Start 2 mounts, both pointing at filer0 for metadata consistency.
// (filer1 exists for the DLM lock ring but both mounts share filer0's
// metadata store since leveldb is per-filer.)
for i := 0; i < 2; i++ {
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
require.NoError(t, os.MkdirAll(mp, 0755))
c.mountPoints[i] = mp
require.NoError(t, c.startMount(i, configDir))
require.NoError(t, c.waitForMount(mp, 30*time.Second),
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
return c
}
func (c *dlmTestCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
// Stop mounts first (triggers flush + DLM unlock).
// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
for i := 1; i >= 0; i-- {
stopCmd(c.mountCmds[i])
// Backup unmount in case FUSE teardown didn't clean up
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
}
// Stop filers, volume, master
for i := 1; i >= 0; i-- {
stopCmd(c.filerCmds[i])
}
stopCmd(c.volumeCmd)
stopCmd(c.masterCmd)
for _, f := range c.logFiles {
f.Close()
}
// Copy logs for CI
c.copyLogsForCI()
if !c.t.Failed() {
os.RemoveAll(c.baseDir)
}
// Wait for ports to be fully released before the next test
// allocates new ports (avoids TIME_WAIT collisions).
time.Sleep(2 * time.Second)
})
}
// masterAddress returns the master address in the format that encodes both
// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
// SeaweedFS uses to communicate non-default gRPC ports between components.
func (c *dlmTestCluster) masterAddress() string {
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
}
func (c *dlmTestCluster) startMaster(configDir string) error {
c.masterCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"master",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.masterPort),
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
"-mdir="+filepath.Join(c.baseDir, "master"),
)
return c.startCmd(c.masterCmd, "master")
}
func (c *dlmTestCluster) startVolume(configDir string) error {
volDir := filepath.Join(c.baseDir, "volume")
if err := os.MkdirAll(volDir, 0755); err != nil {
return fmt.Errorf("create volume dir: %w", err)
}
c.volumeCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"volume",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.volumePort),
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
"-master="+c.masterAddress(),
"-dir="+volDir,
"-max=10",
)
return c.startCmd(c.volumeCmd, "volume")
}
func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
if err := os.MkdirAll(filerDir, 0755); err != nil {
return fmt.Errorf("create filer dir: %w", err)
}
c.filerCmds[idx] = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"filer",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.filerPorts[idx]),
"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
"-master="+c.masterAddress(),
"-filerGroup="+filerGroup,
"-defaultStoreDir="+filerDir,
)
return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
}
func (c *dlmTestCluster) filerAddress(idx int) string {
return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
}
func (c *dlmTestCluster) startMount(idx int, configDir string) error {
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return fmt.Errorf("create cache dir: %w", err)
}
args := []string{
"-logdir=" + filepath.Join(c.baseDir, "logs"),
"mount",
"-filer=" + c.filerAddress(0), // both mounts use filer0 for shared metadata
"-dir=" + c.mountPoints[idx],
"-filer.path=/",
"-dirAutoCreate",
"-allowOthers=false",
"-cacheDir=" + cacheDir,
}
if c.dlm {
args = append(args, "-dlm")
}
c.mountCmds[idx] = exec.Command(c.weedBinary, args...)
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
}
func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
logPath := filepath.Join(c.baseDir, "logs")
if err := os.MkdirAll(logPath, 0755); err != nil {
return fmt.Errorf("create log dir: %w", err)
}
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
if err != nil {
return err
}
c.logFiles = append(c.logFiles, logFile)
cmd.Stdout = logFile
cmd.Stderr = logFile
return cmd.Start()
}
func (c *dlmTestCluster) tailLog(name string) string {
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
if err != nil {
return fmt.Sprintf("(log %s not available: %v)", name, err)
}
const maxTail = 8192
if len(data) > maxTail {
data = data[len(data)-maxTail:]
}
return string(data)
}
func (c *dlmTestCluster) copyLogsForCI() {
// One subdirectory per test: a flat layout lets every teardown overwrite
// the previous test's logs, so the CI artifact only ever shows the last
// cluster, never the failing one.
ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-dlm-logs", strings.ReplaceAll(c.t.Name(), "/", "_"))
os.MkdirAll(ciLogDir, 0755)
logsDir := filepath.Join(c.baseDir, "logs")
entries, err := os.ReadDir(logsDir)
if err != nil {
return
}
for _, e := range entries {
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
if err != nil {
continue
}
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
}
}
func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", addr, time.Second)
if err == nil {
conn.Close()
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("service at %s not ready within timeout", addr)
}
// waitForMount waits for a FUSE filesystem to actually be mounted at
// mountPoint by comparing the device ID of the mount point against its parent.
// A plain directory (pre-created before mount) has the same device as its
// parent; a mounted FUSE filesystem has a different device.
func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
parentDir := filepath.Dir(mountPoint)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
parentStat, err := os.Stat(parentDir)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
mountStat, err := os.Stat(mountPoint)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
parentSys := parentStat.Sys().(*syscall.Stat_t)
mountSys := mountStat.Sys().(*syscall.Stat_t)
if parentSys.Dev != mountSys.Dev {
// Different device = FUSE mounted
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
}
func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
}
func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return err
}
defer conn.Close()
client := master_pb.NewSeaweedClient(conn)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: "filer",
FilerGroup: filerGroup,
})
cancel()
if err == nil && len(resp.ClusterNodes) >= expected {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
}
// waitForVolumeRegistered waits until the volume server shows up in the master
// topology with its slots reported. An open volume port only means the process
// is listening: until the master has elected itself and accepted a heartbeat,
// an assign fails and the first write on a fresh mount surfaces it as ENOSPC.
func (c *dlmTestCluster) waitForVolumeRegistered(timeout time.Duration) error {
addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return err
}
defer conn.Close()
client := master_pb.NewSeaweedClient(conn)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{})
cancel()
if err == nil && resp.TopologyInfo != nil {
var slots int64
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, dn := range rack.DataNodeInfos {
for _, disk := range dn.DiskInfos {
slots += disk.MaxVolumeCount
}
}
}
}
if slots > 0 {
return nil
}
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("volume server not registered with the master within %v", timeout)
}
// waitForLockRingConverged verifies that both filers have a consistent view of
// the lock ring by acquiring the same lock through each filer and checking
// mutual exclusion. Adapted from test/s3/distributed_lock/.
func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
deadline := time.Now().Add(timeout)
owners := []pb.ServerAddress{
pb.ServerAddress(c.filerGRPCAddress(0)),
pb.ServerAddress(c.filerGRPCAddress(1)),
}
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
ring.SetServers(owners)
attempt := 0
for time.Now().Before(deadline) {
testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
attempt++
allConverged := true
for _, key := range testKeys {
converged, _ := c.checkLockMutualExclusion(key)
if !converged {
allConverged = false
break
}
}
if allConverged {
return nil
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("lock ring did not converge")
}
// convergenceKeysPerPrimary generates one test key per primary filer.
func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
found := make(map[pb.ServerAddress]bool)
var keys []string
for i := 0; len(found) < len(owners) && i < 10000; i++ {
key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
primary, _ := ring.GetPrimaryAndBackup(key)
if !found[primary] {
found[primary] = true
keys = append(keys, key)
}
}
return keys
}
func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
// Try to lock via filer0
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn0.Close()
client0 := filer_pb.NewSeaweedFilerClient(conn0)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
Name: key,
SecondsToLock: 5,
Owner: "convergence-test-0",
})
if err != nil {
return false, err
}
if resp0.Error != "" {
return false, fmt.Errorf("lock0: %s", resp0.Error)
}
// Try to lock via filer1 — should fail (already locked)
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn1.Close()
client1 := filer_pb.NewSeaweedFilerClient(conn1)
resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
Name: key,
SecondsToLock: 5,
Owner: "convergence-test-1",
})
// Unlock via filer0
client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
Name: key,
RenewToken: resp0.RenewToken,
})
if err != nil {
return false, err
}
// If filer1 also got the lock, the ring hasn't converged
if resp1.Error == "" {
// Unlock the second one too
client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
Name: key,
RenewToken: resp1.RenewToken,
})
return false, nil
}
return true, nil
}
func stopCmd(cmd *exec.Cmd) {
if cmd == nil || cmd.Process == nil {
return
}
cmd.Process.Signal(syscall.SIGTERM)
done := make(chan struct{})
go func() {
cmd.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
cmd.Process.Kill()
<-done
}
}
func allocatePorts(t testing.TB, n int) []int {
t.Helper()
// Hold all listeners open until all ports are collected, then close
// them together. This prevents the OS from reassigning a just-freed
// port to the next Listen call within the same allocation batch.
listeners := make([]net.Listener, 0, n)
ports := make([]int, 0, n)
defer func() {
for _, l := range listeners {
l.Close()
}
}()
for i := 0; i < n; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
listeners = append(listeners, l)
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
}
return ports
}
func findWeedBinary() string {
if p := os.Getenv("WEED_BINARY"); p != "" {
return p
}
if p, err := exec.LookPath("weed"); err == nil {
return p
}
candidates := []string{
"../../weed/weed",
"./weed",
}
for _, c := range candidates {
if info, err := os.Stat(c); err == nil && !info.IsDir() {
abs, _ := filepath.Abs(c)
return abs
}
}
return ""
}