Files
seaweedfs/weed/command/mount_windows.go
T
Chris LuandClaude Opus 5.5 fd81a50c57 mount: keep retrying writeback chunk uploads through a volume outage (#12465)
* mount: keep retrying writeback chunk uploads through a volume outage

With -writebackCache, close() returns before the data is uploaded. When
the volume servers stay unreachable past the upload's own retry (~20 s),
the chunk was dropped and the file never appeared, with no error to the
application. Retry transient upload errors with the same bytes until they
succeed, unless the dirty pages were discarded. Permanent errors and the
synchronous mode keep failing as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: test writeback upload retries and the cases that still fail

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: keep the filer's read-only verdict on assign failures

UploadWithRetry flattened AssignVolumeResponse errors into a string, so
callers could not tell a read-only rule from an outage. Wrap the filer's
coded error so errors.Is(err, filer_pb.ErrReadOnly) works; the message is
unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: retry writeback uploads the upload layer reassigns, never read-only

A filer proxy that cannot reach the volume answers 500 with an empty
body, which util.IsTransientError does not match, so that outage still
dropped the chunk. Also retry what operation.ShouldReassignUpload takes.
A read-only rule stays final even when its path or message contains a
transient word such as "unavailable".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: stop a writeback upload once its dirty pages are discarded

Destroy() was only seen after a failed attempt, so a truncate during the
retry wait still started another upload, and a chunk that finished after
its pages were replaced was attached to the new content. Check before
each attempt, and drop the chunk instead of attaching it; volume.fsck
reclaims it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: keep Destroy from slipping between the discard check and the append

truncateEntry destroys the pages and then installs the new entry, so a
Destroy landing after the check but before AddChunks put the old chunk
into the replacing entry. Hold one lock across the check and the append,
and take it in Destroy while setting the flag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: leave the Windows interrupt drain to the capped filesystem hook

Interrupt hooks run in reverse registration order, so the Windows hook ran
before the filesystem hook and waited on pending writeback flushes without a
bound. With writeback uploads now retrying through an outage, Ctrl-C could
wait until the volume servers came back. Unmount only, as on Linux, and let
the filesystem hook drain for at most 30 s and keep the write cache.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: test a writeback upload whose assign keeps timing out

With every volume server gone the assign RPC times out until the filer
client gives up; that error must be retried like a refused upload.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 19:24:42 -08:00

206 lines
6.9 KiB
Go

package command
import (
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
"github.com/seaweedfs/seaweedfs/weed/mount/winfsp"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
// ownedByMounter reports the mount root as belonging to whoever started it,
// matching the uid=-1 option handed to WinFsp. It is a display value only:
// WinFsp substitutes the calling user, and it must never be persisted, since
// every other client would read the entry as owned by uid 4294967295.
const ownedByMounter = ^uint32(0)
// windowsCacheTimeout bounds how long the adapter serves cached paths and
// attributes and WinFsp serves cached directory listings, matching what the
// kernel caches would be allowed on a unix mount. Metadata events purge
// entries earlier; this is the backstop for anything an event misses.
const windowsCacheTimeout = time.Second
func RunMount(option *MountOptions, umask os.FileMode) bool {
if err := configureMountMemory(option); err != nil {
fmt.Println(err)
return false
}
chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
if chunkSizeLimitMB <= 0 {
fmt.Printf("Please specify a reasonable buffer size.\n")
return false
}
dir := *option.dir
if dir == "" {
fmt.Printf("Please specify the mount point via \"-dir\", for example -dir=S:\n")
return false
}
// A drive letter or a not-yet-existing directory is what WinFsp wants, so
// the mount point deliberately goes through none of the unix preparation:
// no ResolvePath, no auto-create, no stat.
if err := checkWindowsMountPoint(dir); err != nil {
fmt.Printf("%v\n", err)
return false
}
filerAddresses, grpcDialOption, cipher, bucketRootPath, ok := connectToFiler(option)
if !ok {
return true
}
if *option.localSocket == "" {
mountDirHash := util.HashToInt32([]byte(dir))
if mountDirHash < 0 {
mountDirHash = -mountDirHash
}
*option.localSocket = filepath.Join(os.TempDir(), fmt.Sprintf("seaweedfs-mount-%d.sock", mountDirHash))
}
if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
}
mountSocketListener, err := net.Listen("unix", *option.localSocket)
if err != nil {
glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
}
uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap)
if err != nil {
fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err)
return false
}
mountRoot := resolveMountRoot(*option.filerMountRootPath)
cacheDirForRead, cacheDirForWrite := resolveCacheDirs(option)
seaweedFileSystem := buildSeaweedFileSystem(option, fileSystemParams{
dir: dir,
mountRoot: mountRoot,
filerAddresses: filerAddresses,
grpcDialOption: grpcDialOption,
cipher: cipher,
uidGidMapper: uidGidMapper,
uid: uint32(*option.windowsUid),
gid: uint32(*option.windowsGid),
mountMode: os.ModeDir | 0777,
mountCtime: time.Now(),
umask: umask,
chunkSizeLimitMB: chunkSizeLimitMB,
cacheDirForRead: cacheDirForRead,
cacheDirForWrite: cacheDirForWrite,
// WinFsp posts the cleanup and close that carry the flush after
// CloseHandle has already returned, so entry creation cannot wait
// for the flush the way it does when close(2) runs it synchronously.
eagerFilerCreate: true,
})
if !createMountRoot(seaweedFileSystem, mountRoot, bucketRootPath, filerAddresses) {
return false
}
host := winfsp.New(seaweedFileSystem, winfsp.Options{
VolumeName: volumeName(*option.filer, *option.filerMountRootPath, dir, *option.volumeName),
Uid: ownedByMounter,
Gid: ownedByMounter,
CacheTimeout: windowsCacheTimeout,
ReadOnly: *option.readOnly,
Debug: *option.debugFuse,
ExtraOptions: option.extraOptions,
})
// Windows caches entries on its own side and the mount cannot invalidate
// that cache directly, so changes made elsewhere have to be pushed out.
host.Notify(seaweedFileSystem)
grace.OnInterrupt(func() {
// The filesystem's own hook runs next and drains queued writeback
// flushes for at most 30 s, keeping the write cache if they are not done.
host.Unmount()
})
serveMountGrpc(seaweedFileSystem, mountSocketListener)
if err := seaweedFileSystem.StartBackgroundTasks(); err != nil {
fmt.Printf("failed to start background tasks: %v\n", err)
return false
}
glog.V(0).Infof("mounting %s%s to %v", *option.filer, mountRoot, dir)
glog.V(0).Infof("This is SeaweedFS version %s %s %s", version.Version(), runtime.GOOS, runtime.GOARCH)
glog.V(0).Infof("Windows mount is beta: hard links are unavailable and byte-range locks are not shared across mounts")
if prefix, _ := winfsp.VolumePrefix(dir); prefix != "" {
glog.V(0).Infof(`network mount: every session on this machine can open \%s or map a drive letter to it`, prefix)
}
if err := host.Serve(windowsMountPoint(dir)); err != nil {
glog.Errorf("%v", err)
return false
}
seaweedFileSystem.WaitForAsyncFlush()
seaweedFileSystem.ClearCacheDir()
return true
}
// checkWindowsMountPoint rejects the mount points WinFsp cannot take, which is
// worth doing up front because its own failure is a bare false.
func checkWindowsMountPoint(dir string) error {
if isDriveLetter(dir) {
if _, err := os.Stat(strings.TrimRight(dir, `\/`) + `\`); err == nil {
return fmt.Errorf("drive %s is already in use", dir)
}
return nil
}
if prefix, err := winfsp.VolumePrefix(dir); err != nil || prefix != "" {
return err
}
if strings.HasPrefix(dir, `\\`) {
return nil
}
// WinFsp creates the directory itself, with FILE_CREATE, and deletes it
// again when the filesystem goes away. An existing one — empty or not —
// fails with "mount point in use".
if _, err := os.Stat(dir); err == nil {
return fmt.Errorf("mount point %s already exists; WinFsp creates the directory itself, so give it a path that does not exist yet", dir)
} else if !os.IsNotExist(err) {
return err
}
if _, err := os.Stat(filepath.Dir(dir)); err != nil {
return fmt.Errorf("parent of mount point %s does not exist", dir)
}
return nil
}
// windowsMountPoint drops a trailing separator from a drive letter. WinFsp
// recognises a drive only as exactly two characters; "S:\\" falls through to
// its directory handling and the mount fails.
func windowsMountPoint(dir string) string {
if isDriveLetter(dir) {
return strings.TrimRight(dir, `\/`)
}
return dir
}
// isDriveLetter accepts both "S:" and "S:\\"; the trailing separator is how
// the drive is usually written, and windowsMountPoint normalises it.
func isDriveLetter(dir string) bool {
trimmed := strings.TrimRight(dir, `\/`)
if len(trimmed) != 2 || trimmed[1] != ':' {
return false
}
c := trimmed[0]
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
}