From c76fb36416c01803f313802e728b0300bcf9ade1 Mon Sep 17 00:00:00 2001 From: Yip Leo Date: Wed, 18 Nov 2026 09:08:39 +0800 Subject: [PATCH] test(fuse): two mounts staging the same files without -dlm (#12704) * test(fuse): two mounts staging the same files without -dlm Two jobs that extract the same archive into one shared directory, each through its own mount and without -dlm, both create many of the same files: a file that exists is skipped, otherwise it is created, written and closed, then chmod'ed and given its mtime. Before #12593, #12594 and #12596 the attribute update and the content flush sent the mount's whole cached entry. Sent after the other mount had rewritten the file, the update put back chunks that the rewrite had dropped and the filer had deleted, so the entry pointed at deleted needles and the file failed to read once its volume was vacuumed. Nothing told the writers. TestStageSameFilesWithoutDLM runs that on the fuse_dlm cluster with the mounts started without -dlm (startTestCluster takes a dlm flag; startDLMTestCluster is unchanged) and checks every staged file: each chunk is still on the volume server (HEAD on the fid, without readDeleted) and the filer serves the bytes written. What the losing writer is told depends on the version and is only logged. With a 4.53 weed binary the test fails: 66, 71 and 70 of the 400 files were broken in three runs. With master it passes in about 48 s. Co-Authored-By: Claude Opus 5.5 * test(fuse): bound the staging test's filer and volume checks The checks used the default HTTP client and a background context, so a server that accepts a request and stops answering held the test until the package timeout instead of failing the check. Co-Authored-By: Claude Opus 5.5 * test(fuse): fail the staging test unless both mounts wrote the same files A mount whose MkdirAll failed returned early, and a mount that started late could skip every file, so the test passed with one writer. Start both loops together, fail on setup errors, and require files both mounts wrote. Co-Authored-By: Claude Opus 5.5 * test(fuse): bound the staging test's checks with one deadline A per-call timeout still let 400 hung lookups and reads run for hours. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- test/fuse_dlm/framework_test.go | 27 ++-- test/fuse_dlm/stage_without_dlm_test.go | 177 ++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 8 deletions(-) create mode 100644 test/fuse_dlm/stage_without_dlm_test.go diff --git a/test/fuse_dlm/framework_test.go b/test/fuse_dlm/framework_test.go index fbc76414f2..8660aabfcc 100644 --- a/test/fuse_dlm/framework_test.go +++ b/test/fuse_dlm/framework_test.go @@ -28,11 +28,13 @@ import ( const filerGroup = "fuse-dlm-test" // dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE -// mounts for testing DLM-based cross-mount write coordination. +// mounts for testing DLM-based cross-mount write coordination, and what two +// mounts do without it. type dlmTestCluster struct { t testing.TB baseDir string weedBinary string + dlm bool // the mounts run with -dlm masterPort int masterGrpcPort int @@ -52,6 +54,11 @@ type dlmTestCluster struct { } func startDLMTestCluster(t testing.TB) *dlmTestCluster { + return startTestCluster(t, true) +} + +// startTestCluster starts the cluster; dlm says whether the mounts run with -dlm. +func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster { binary := findWeedBinary() if binary == "" { t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH") @@ -64,6 +71,7 @@ func startDLMTestCluster(t testing.TB) *dlmTestCluster { t: t, baseDir: baseDir, weedBinary: binary, + dlm: dlm, } // Register cleanup early so processes are stopped even if a require fails below. t.Cleanup(c.Stop) @@ -224,17 +232,20 @@ func (c *dlmTestCluster) startMount(idx int, configDir string) error { if err := os.MkdirAll(cacheDir, 0755); err != nil { return fmt.Errorf("create cache dir: %w", err) } - c.mountCmds[idx] = exec.Command(c.weedBinary, - "-logdir="+filepath.Join(c.baseDir, "logs"), + args := []string{ + "-logdir=" + filepath.Join(c.baseDir, "logs"), "mount", - "-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata - "-dir="+c.mountPoints[idx], + "-filer=" + c.filerAddress(0), // both mounts use filer0 for shared metadata + "-dir=" + c.mountPoints[idx], "-filer.path=/", "-dirAutoCreate", "-allowOthers=false", - "-cacheDir="+cacheDir, - "-dlm", - ) + "-cacheDir=" + cacheDir, + } + if c.dlm { + args = append(args, "-dlm") + } + c.mountCmds[idx] = exec.Command(c.weedBinary, args...) return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx)) } diff --git a/test/fuse_dlm/stage_without_dlm_test.go b/test/fuse_dlm/stage_without_dlm_test.go new file mode 100644 index 0000000000..806743c6f0 --- /dev/null +++ b/test/fuse_dlm/stage_without_dlm_test.go @@ -0,0 +1,177 @@ +//go:build linux || darwin + +package fuse_dlm + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// TestStageSameFilesWithoutDLM has two mounts without -dlm stage the same +// files into one directory at the same time, the way two jobs extract the same +// archive into a shared directory: a file that exists is skipped, otherwise it +// is created, written and closed, then chmod'ed and given its mtime. Both +// mounts often create the same file, and each then changes the attributes of +// the copy it has cached. +// +// Whatever either mount is told, no entry may point at a chunk that is gone +// from the volume server, and every file must read back whole. An update sent +// from a cached copy after the other mount rewrote the file puts back the +// chunks the rewrite dropped, which the filer deletes; reads of such a file +// fail once the volume is vacuumed. +func TestStageSameFilesWithoutDLM(t *testing.T) { + if testing.Short() { + t.Skip("skipping FUSE integration test in short mode") + } + + cluster := startTestCluster(t, false) + t.Cleanup(cluster.Stop) + + const ( + files = 400 + dir = "staged" + ) + rng := rand.New(rand.NewSource(1)) + names := make([]string, files) + content := make(map[string][]byte, files) + for i := range names { + names[i] = fmt.Sprintf("img-%04d.jpg", i) + data := make([]byte, 1024+rng.Intn(63*1024)) + rng.Read(data) + content[names[i]] = data + } + mtime := time.Date(2026, 10, 26, 4, 0, 0, 0, time.UTC) + + // What the losing mount is told depends on the version (nothing, EIO, + // EAGAIN); the files are what this test checks. + failed := make([][]string, 2) + setupErr := make([]error, 2) + wrote := [2][]bool{make([]bool, files), make([]bool, files)} + var ready, wg sync.WaitGroup + ready.Add(2) + for m := 0; m < 2; m++ { + wg.Add(1) + go func(m int) { + defer wg.Done() + stage := filepath.Join(cluster.mountPoints[m], dir) + setupErr[m] = os.MkdirAll(stage, 0755) + ready.Done() + ready.Wait() + if setupErr[m] != nil { + return + } + for i, name := range names { + p := filepath.Join(stage, name) + if _, err := os.Lstat(p); err == nil { + continue + } + wrote[m][i] = true + if err := os.WriteFile(p, content[name], 0644); err != nil { + failed[m] = append(failed[m], err.Error()) + continue + } + if err := os.Chmod(p, 0640); err != nil { + failed[m] = append(failed[m], err.Error()) + continue + } + if err := os.Chtimes(p, mtime, mtime); err != nil { + failed[m] = append(failed[m], err.Error()) + } + } + }(m) + } + wg.Wait() + for m, f := range failed { + require.NoError(t, setupErr[m], "mount%d", m) + if len(f) > 0 { + t.Logf("mount%d: %d calls failed, first: %s", m, len(f), f[0]) + } + } + both := 0 + for i := range names { + if wrote[0][i] && wrote[1][i] { + both++ + } + } + t.Logf("both mounts wrote %d of %d files", both, files) + require.Positive(t, both, "the mounts never wrote the same file") + + // The filer deletes the chunks an update dropped shortly after the update. + time.Sleep(5 * time.Second) + + conn, err := grpc.NewClient(cluster.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + client := filer_pb.NewSeaweedFilerClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + var broken []string + for _, name := range names { + if ctx.Err() != nil { + broken = append(broken, "checks did not finish: "+ctx.Err().Error()) + break + } + resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: "/" + dir, + Name: name, + }) + if err != nil { + broken = append(broken, fmt.Sprintf("%s: lookup: %v", name, err)) + continue + } + for _, chunk := range resp.Entry.GetChunks() { + if !cluster.needleExists(ctx, t, chunk.GetFileIdString()) { + broken = append(broken, fmt.Sprintf("%s: chunk %s is deleted", name, chunk.GetFileIdString())) + } + } + if got, err := cluster.readFromFiler(ctx, "/"+dir+"/"+name); err != nil { + broken = append(broken, fmt.Sprintf("%s: read: %v", name, err)) + } else if !bytes.Equal(got, content[name]) { + broken = append(broken, fmt.Sprintf("%s: read %d bytes, want the %d written", name, len(got), len(content[name]))) + } + } + require.Empty(t, broken, "%d of %d staged files are broken", len(broken), files) +} + +// needleExists reports whether the volume server still has the needle; a +// deleted one answers 404 (only readDeleted=true would serve it). +func (c *dlmTestCluster) needleExists(ctx context.Context, t testing.TB, fid string) bool { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("http://127.0.0.1:%d/%s", c.volumePort, fid), nil) + require.NoError(t, err) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + resp.Body.Close() + return resp.StatusCode == http.StatusOK +} + +// readFromFiler reads a file whole through filer0's HTTP API. +func (c *dlmTestCluster) readFromFiler(ctx context.Context, path string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPorts[0], path), nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status %s", resp.Status) + } + return io.ReadAll(resp.Body) +}