mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* mount: ask the filer again when a deferred create is published mid-lookup A program that writes a file under a temporary name and renames it while the file is still open (safetensors save_file does, through Rust tempfile's persist) looks the temporary name up before the rename. When the parent directory's listing is not cached, that lookup goes to the filer. If a FLUSH publishes the deferred create while the lookup is in flight (a forked child closing its inherited descriptor sends one), the filer can answer not-found from a read taken before the create committed. By the time the answer arrives the flush has cleared the handle's dirty flag, so lookupEntry finds nothing local to explain the answer, logs "possible coherence bug" and returns ENOENT, and the rename fails although the file exists. Note before sending the request whether the inode had unpublished metadata (a dirty handle or an async flush in flight). On not-found, if it had and has since been published, ask the filer once more: the filer acknowledged the publish before the second request is sent. A second not-found stands, so a file another mount removed is not brought back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: judge a placeholder's not-found by one look at its base Review of the previous commit found two problems. The retry check and the not-found fallback read the handle separately, so a flush that finished between the two reads still turned the answer into ENOENT. And the pre-request read of dirtyMetadata, a plain bool that Write and the flush set under the handle lock, raced with them on every lookup that reached the filer. Note instead whether the handle is a local placeholder: a deferred create has no filer base (baseEntry) until a flush publishes it. A not-found for a placeholder is judged by one atomic look at that base. With none, the filer cannot have the file yet and the handle answers. With a base, the filer acknowledged the publish, so ask once more. dirtyMetadata is not read for this, and the not-found fallback keeps the existing check, with the placeholder decision added. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: answer a placeholder from its own handle; a second miss is final Follow-ups from review of the previous commit: - Pick the placeholder after the async-flush wait, from the path's current mapping. - A placeholder that still has no base answers from the observed handle itself, instead of through the not-found fallback, which re-resolved the path and could miss once the handle was released and the cache entry evicted. - A not-found from the second request is final. The flush installs the base before it clears dirtyMetadata, and keeps it set when a write landed meanwhile, so the fallback could return a file another mount had removed. - Test gates release on cleanup, so a failing test leaves no handler blocked. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: read a placeholder's entry the way GetAttr does The previous commit answered a not-found for a placeholder from the handle's live entry under entryLock. Neither of the entry's writers takes that lock: Write and the metadata flush change size, times and chunks under the handle lock, and async upload workers append chunks under the LockedEntry lock, so the lookup could read an entry while it changed. Answer with the copy the create cached when it is still this file's (same inode), as the not-found fallback already prefers. Otherwise copy the handle's entry under both writers' locks, in GetAttr's order (handle lock shared, then the LockedEntry lock), and convert the copy. No caller of lookupEntry holds a handle lock. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: cache a deferred create's placeholder in filer ids The meta cache stores filer uid/gid and maps them to local ids on every read, but createRegularFile cached the placeholder with the caller's local ids. Under a uid/gid map whose local id is also a mapped filer id, a lookup or stat answered from the placeholder reported the wrong owner. Cache a mapped clone and keep the handle's entry in local ids. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: assert the placeholder is cached before checking its owner Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: check the snapshot owner after evicting the cached placeholder Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
401 lines
14 KiB
Go
401 lines
14 KiB
Go
package mount
|
|
|
|
import (
|
|
"bytes"
|
|
"os"
|
|
"sync"
|
|
"sync/atomic"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
"github.com/seaweedfs/go-fuse/v2/fuse"
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
type FileHandleId uint64
|
|
|
|
var IsDebugFileReadWrite = false
|
|
|
|
type FileHandle struct {
|
|
fh FileHandleId
|
|
counter int64
|
|
fhs map[FileHandleId]bool // live fh values, one per open; guarded by fhMap's lock
|
|
entry *LockedEntry
|
|
// Nested handle lock order: wfs.fhLockTable[fh.fh] -> entryLock ->
|
|
// LockedEntry.RWMutex. Paths may use a subset, but must never acquire an
|
|
// earlier lock while holding a later one. Release in reverse order.
|
|
entryLock sync.RWMutex
|
|
entryChunkGroup *filer.ChunkGroup
|
|
inode uint64
|
|
wfs *WFS
|
|
|
|
// cache file has been written to
|
|
pendingAppend bool
|
|
pendingAppendOffset uint64
|
|
writebackContentError *atomic.Bool
|
|
dirtyMetadata bool
|
|
dirtyContent bool
|
|
dirtyPages *PageWriter
|
|
reader *filer.ChunkReadAt
|
|
contentType string
|
|
asyncFlushPending bool // set in writebackCache mode to defer flush to Release
|
|
asyncFlushUid uint32 // saved uid for deferred metadata flush
|
|
asyncFlushGid uint32 // saved gid for deferred metadata flush
|
|
savedDir string // last known parent path if inode-to-path state is forgotten
|
|
savedName string // last known file name if inode-to-path state is forgotten
|
|
|
|
isDeleted bool
|
|
// deleteEpoch counts the times isDeleted was raised, all of them under the
|
|
// handle's flush lock. A caller that raised it and then found it had
|
|
// nothing to delete after all can tell its own mark from a later one.
|
|
deleteEpoch uint64
|
|
isRenamed bool // set by Rename before waiting for async flush; skips old-path metadata flush
|
|
// linkRefresh is the hard link id of a link change whose stored entry
|
|
// could not be read; the next flush reads it before publishing.
|
|
linkRefresh []byte
|
|
|
|
// pageWrites counts the pages added to dirtyPages, under the handle lock.
|
|
pageWrites atomic.Uint64
|
|
|
|
// entryVersionTsNs is the filer log position the handle's entry reflects.
|
|
// State at or below it must not replace the entry — that rolls it back.
|
|
entryVersionTsNs atomic.Int64
|
|
|
|
// entryVersionSignature identifies the filer whose clock stamped
|
|
// entryVersionTsNs, when it came from an RPC fence. Positions from a
|
|
// different filer are not comparable, so an event that filer did not log
|
|
// is applied rather than fenced out. Zero when the version came from an
|
|
// event, whose ordering the subscription already provides.
|
|
entryVersionSignature atomic.Int32
|
|
|
|
// baseEntry snapshots the filer state last installed or acknowledged.
|
|
// Local writes move the live entry away from it, so "is this event new"
|
|
// must be judged here, not against the live entry. Always store a clone.
|
|
baseEntry atomic.Pointer[filer_pb.Entry]
|
|
|
|
// sinceBase marks a truncate or chunk rewrite over baseEntry that a
|
|
// merge must honor: later writes can grow the file past a truncate, and
|
|
// rewritten chunks carry the handle's view of the base.
|
|
sinceBase atomic.Pointer[baseMark]
|
|
|
|
// chunkViewStale marks a read view whose manifest resolution failed; the
|
|
// next read or lseek resolves the entry's chunks again.
|
|
chunkViewStale atomic.Bool
|
|
|
|
// dlmLock holds the distributed lock for cross-mount write coordination.
|
|
// Non-nil only when -dlm is enabled and the file was opened for writing.
|
|
// Acquired in AcquireHandle, released in ReleaseHandle.
|
|
dlmLock *cluster.LiveLock
|
|
|
|
// remoteInstallMu serializes downloadRemoteEntry's install, which holds
|
|
// only the handle's shared lock and so races a second concurrent read.
|
|
remoteInstallMu sync.Mutex
|
|
|
|
// for debugging
|
|
mirrorFile *os.File
|
|
}
|
|
|
|
func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle {
|
|
fh := &FileHandle{
|
|
fh: handleId,
|
|
counter: 1,
|
|
fhs: map[FileHandleId]bool{handleId: false},
|
|
inode: inode,
|
|
wfs: wfs,
|
|
}
|
|
// dirtyPages: newContinuousDirtyPages(file, writeOnly),
|
|
fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
|
|
fh.entry = &LockedEntry{
|
|
Entry: entry,
|
|
}
|
|
if entry != nil {
|
|
fh.SetEntry(entry)
|
|
fh.baseEntry.Store(proto.Clone(entry).(*filer_pb.Entry))
|
|
}
|
|
|
|
if IsDebugFileReadWrite {
|
|
var err error
|
|
fh.mirrorFile, err = os.OpenFile("/tmp/sw/"+entry.Name, os.O_RDWR|os.O_CREATE, 0600)
|
|
if err != nil {
|
|
println("failed to create mirror:", err.Error())
|
|
}
|
|
}
|
|
|
|
return fh
|
|
}
|
|
|
|
func (fh *FileHandle) FullPath() util.FullPath {
|
|
if fp, status := fh.wfs.inodeToPath.GetPath(fh.inode); status == fuse.OK {
|
|
return fp
|
|
}
|
|
if fh.savedName != "" {
|
|
return util.FullPath(fh.savedDir).Child(fh.savedName)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (fh *FileHandle) RememberPath(fullPath util.FullPath) {
|
|
if fullPath == "" {
|
|
return
|
|
}
|
|
fh.savedDir, fh.savedName = fullPath.DirAndName()
|
|
}
|
|
|
|
func (fh *FileHandle) GetEntry() *LockedEntry {
|
|
return fh.entry
|
|
}
|
|
|
|
// entrySnapshot copies the handle's entry under the locks of both of its
|
|
// writers, as GetAttr reads it: Write and the metadata flush change size,
|
|
// times and chunks under the handle lock, while async upload workers append
|
|
// chunks under the LockedEntry lock.
|
|
func (fh *FileHandle) entrySnapshot(dir string) *filer.Entry {
|
|
fhActiveLock := fh.wfs.fhLockTable.AcquireLock("entrySnapshot", fh.fh, util.SharedLock)
|
|
defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
|
|
fh.entry.RLock()
|
|
var snapshot *filer_pb.Entry
|
|
if fh.entry.Entry != nil {
|
|
snapshot = proto.Clone(fh.entry.Entry).(*filer_pb.Entry)
|
|
}
|
|
fh.entry.RUnlock()
|
|
if snapshot == nil {
|
|
return nil
|
|
}
|
|
return filer.FromPbEntry(dir, snapshot)
|
|
}
|
|
|
|
func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) {
|
|
if entry != nil {
|
|
fileSize := filer.FileSize(entry)
|
|
entry.Attributes.FileSize = fileSize
|
|
if fh.entryChunkGroup != nil {
|
|
_ = fh.entryChunkGroup.Close()
|
|
}
|
|
var resolveManifestErr error
|
|
fh.entryChunkGroup, resolveManifestErr = filer.NewChunkGroup(fh.wfs.LookupFn(), fh.wfs.chunkCache, entry.Chunks, fh.wfs.option.ConcurrentReaders, fh.wfs.CacheInvalidator(), fh.wfs.manifestCache, fh.wfs.readerCacheBudget)
|
|
if resolveManifestErr != nil {
|
|
glog.Warningf("failed to resolve manifest chunks in %+v", filer_pb.ScrubCipherKeys(entry))
|
|
}
|
|
fh.chunkViewStale.Store(resolveManifestErr != nil)
|
|
} else {
|
|
glog.Fatalf("setting file handle entry to nil")
|
|
}
|
|
fh.entry.SetEntry(entry)
|
|
}
|
|
|
|
// installAckedEntry installs filer-acknowledged state under the handle lock
|
|
// when it outranks the handle. A version never advances without its value:
|
|
// stamping alone would fence out the events carrying what the handle lacks.
|
|
// Dirty handles are skipped — local writes supersede the ack. Deleted ones
|
|
// too: their path may already name a different file, and no acknowledged
|
|
// state belongs to this handle anymore.
|
|
func (fh *FileHandle) installAckedEntry(entry *filer_pb.Entry, versionTsNs int64, signature int32) {
|
|
fhActiveLock := fh.wfs.fhLockTable.AcquireLock("installAckedEntry", fh.fh, util.ExclusiveLock)
|
|
defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
|
|
if versionTsNs == 0 || fh.dirtyMetadata || fh.isDeleted || entry == fh.GetEntry().GetEntry() {
|
|
return
|
|
}
|
|
// Refuse only what is provably older. Two known, differing filer
|
|
// signatures mean the positions come from unrelated clocks and say
|
|
// nothing about each other; dropping the acknowledgment there would leave
|
|
// the handle holding the very state this mutation replaced. Unknown
|
|
// signatures still compare, as they did before.
|
|
handleSignature := fh.entryVersionSignature.Load()
|
|
provablyOtherClock := signature != 0 && handleSignature != 0 && signature != handleSignature
|
|
if !provablyOtherClock && versionTsNs <= fh.entryVersionTsNs.Load() {
|
|
return
|
|
}
|
|
fh.SetEntry(entry)
|
|
fh.setAuthoritativeBase(proto.Clone(entry).(*filer_pb.Entry))
|
|
fh.advanceEntryVersion(versionTsNs, signature)
|
|
}
|
|
|
|
// rebaseEntry installs an entry merged with a peer's changes, keeping the
|
|
// chunks and growth writers added after sent was taken, and refreshes the
|
|
// read view.
|
|
func (fh *FileHandle) rebaseEntry(merged, sent *filer_pb.Entry) {
|
|
inSent := make(map[string]bool, len(sent.Chunks))
|
|
for _, c := range sent.Chunks {
|
|
inSent[contentChunkKey(c)] = true
|
|
}
|
|
merged = proto.Clone(merged).(*filer_pb.Entry)
|
|
fh.UpdateEntry(func(e *filer_pb.Entry) {
|
|
chunks := merged.Chunks
|
|
for _, c := range e.Chunks {
|
|
if !inSent[contentChunkKey(c)] {
|
|
chunks = append(chunks, c)
|
|
}
|
|
}
|
|
attrs := merged.Attributes
|
|
if size := e.GetAttributes().GetFileSize(); size > sent.GetAttributes().GetFileSize() && size > attrs.FileSize {
|
|
attrs.FileSize = size
|
|
}
|
|
e.Attributes, e.Extended, e.Content, e.Chunks = attrs, merged.Extended, merged.Content, chunks
|
|
err := fh.entryChunkGroup.SetChunks(chunks)
|
|
if err != nil {
|
|
glog.Warningf("rebase %s: resolve chunks: %v", fh.FullPath(), err)
|
|
}
|
|
fh.chunkViewStale.Store(err != nil)
|
|
})
|
|
}
|
|
|
|
// refreshChunkView retries a read view whose manifest resolution failed.
|
|
func (fh *FileHandle) refreshChunkView() {
|
|
if !fh.chunkViewStale.Load() {
|
|
return
|
|
}
|
|
fh.UpdateEntry(func(e *filer_pb.Entry) {
|
|
fh.chunkViewStale.Store(fh.entryChunkGroup.SetChunks(e.Chunks) != nil)
|
|
})
|
|
}
|
|
|
|
// mergePeerContent folds a peer's change into a dirty handle, so its own
|
|
// writes stay on top of the peer's instead of being discarded.
|
|
func (fh *FileHandle) mergePeerContent(peer, base *filer_pb.Entry) bool {
|
|
var local *filer_pb.Entry
|
|
fh.UpdateEntry(func(e *filer_pb.Entry) {
|
|
local = proto.Clone(e).(*filer_pb.Entry)
|
|
})
|
|
mark := fh.markSince(base)
|
|
merged, ok := mergeContentEntry(peer, base, local, local, mark.truncatedTo, mark.rewrote)
|
|
ok = ok && mergeKeepsBuffered(merged, local)
|
|
if ok {
|
|
fh.rebaseEntry(merged, local)
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// mergeKeepsBuffered reports whether merged leaves room for writes still
|
|
// buffered over local. They are not chunks yet, so a merge that shrinks the
|
|
// file or brings in inline content (read ahead of chunks) waits for the flush.
|
|
func mergeKeepsBuffered(merged, local *filer_pb.Entry) bool {
|
|
return merged.Attributes.FileSize >= filer.FileSize(local) && bytes.Equal(merged.Content, local.Content)
|
|
}
|
|
|
|
// baseMark records what the handle did over base that its entry does not
|
|
// show: the lowest size a truncate set (-1 for none), and a whole-file
|
|
// rewrite of its chunks.
|
|
type baseMark struct {
|
|
base *filer_pb.Entry
|
|
truncatedTo int64
|
|
rewrote bool
|
|
}
|
|
|
|
func (fh *FileHandle) markBase(update func(m *baseMark)) {
|
|
for {
|
|
base, old := fh.baseEntry.Load(), fh.sinceBase.Load()
|
|
next := baseMark{base: base, truncatedTo: -1}
|
|
if old != nil && old.base == base {
|
|
next = *old
|
|
}
|
|
update(&next)
|
|
if fh.sinceBase.CompareAndSwap(old, &next) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// noteTruncate records a truncate to size over the current base.
|
|
func (fh *FileHandle) noteTruncate(size uint64) {
|
|
fh.markBase(func(m *baseMark) {
|
|
if m.truncatedTo < 0 || int64(size) < m.truncatedTo {
|
|
m.truncatedTo = int64(size)
|
|
}
|
|
})
|
|
}
|
|
|
|
// noteRewrite records that the handle's chunks were rewritten from its view
|
|
// of the current base.
|
|
func (fh *FileHandle) noteRewrite() {
|
|
fh.markBase(func(m *baseMark) { m.rewrote = true })
|
|
}
|
|
|
|
// adoptPeerBase moves a dirty handle's base to a peer's entry. What the
|
|
// handle has not published yet still applies over the new base.
|
|
func (fh *FileHandle) adoptPeerBase(base *filer_pb.Entry) {
|
|
old := fh.baseEntry.Swap(base)
|
|
if m := fh.sinceBase.Load(); m != nil && m.base == old {
|
|
moved := *m
|
|
moved.base = base
|
|
fh.sinceBase.CompareAndSwap(m, &moved)
|
|
}
|
|
}
|
|
|
|
// markSince is what the handle did over base.
|
|
func (fh *FileHandle) markSince(base *filer_pb.Entry) baseMark {
|
|
if m := fh.sinceBase.Load(); m != nil && m.base == base {
|
|
return *m
|
|
}
|
|
return baseMark{base: base, truncatedTo: -1}
|
|
}
|
|
|
|
// setAuthoritativeBase installs the base snapshot a local ack acknowledged.
|
|
func (fh *FileHandle) setAuthoritativeBase(base *filer_pb.Entry) {
|
|
fh.baseEntry.Store(base)
|
|
}
|
|
|
|
// advanceEntryVersion raises the entry version, never regresses it, and
|
|
// records the clock domain the new position belongs to: a filer signature for
|
|
// an RPC fence, zero for an event. The signature travels with the timestamp so
|
|
// the two never disagree. A zero position (an unversioned old filer) is a
|
|
// no-op, leaving the handle open to refreshes.
|
|
func (fh *FileHandle) advanceEntryVersion(tsNs int64, signature int32) {
|
|
if tsNs == 0 {
|
|
return
|
|
}
|
|
for {
|
|
current := fh.entryVersionTsNs.Load()
|
|
if tsNs <= current {
|
|
return
|
|
}
|
|
if fh.entryVersionTsNs.CompareAndSwap(current, tsNs) {
|
|
fh.entryVersionSignature.Store(signature)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (fh *FileHandle) ResetDirtyPages() {
|
|
fh.dirtyPages.Destroy()
|
|
fh.dirtyPages = newPageWriter(fh, fh.wfs.option.ChunkSizeLimit)
|
|
fh.dirtyMetadata = false
|
|
fh.dirtyContent = false
|
|
fh.pendingAppend = false
|
|
fh.contentType = ""
|
|
}
|
|
|
|
func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entry {
|
|
result := fh.entry.UpdateEntry(fn)
|
|
|
|
return result
|
|
}
|
|
|
|
func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) {
|
|
fh.entry.AppendChunks(chunks)
|
|
}
|
|
|
|
func (fh *FileHandle) ReleaseHandle() {
|
|
fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock)
|
|
defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
|
|
|
|
// Release distributed lock before cleaning up, so other mounts can
|
|
// proceed as soon as this handle is done flushing.
|
|
if fh.dlmLock != nil {
|
|
fh.dlmLock.Stop()
|
|
fh.dlmLock = nil
|
|
glog.V(1).Infof("DLM lock released for inode %d", fh.inode)
|
|
}
|
|
|
|
if fh.entryChunkGroup != nil {
|
|
_ = fh.entryChunkGroup.Close()
|
|
}
|
|
fh.dirtyPages.Destroy()
|
|
if IsDebugFileReadWrite {
|
|
fh.mirrorFile.Close()
|
|
}
|
|
}
|