From de5b6f21205345f7602c6031a6ae47963470a1c7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 10 Apr 2026 11:51:06 -0700 Subject: [PATCH] fix(filer,mount): add nanosecond timestamp precision (#9019) * fix(filer,mount): add nanosecond timestamp precision Add mtime_ns and ctime_ns fields to the FuseAttributes protobuf message to store the nanosecond component of timestamps (0-999999999). Previously timestamps were truncated to whole seconds. - Update EntryAttributeToPb/PbToEntryAttribute to encode/decode ns - Update setAttrByPbEntry/setAttrByFilerEntry to set Mtimensec/Ctimensec - Update in-memory atime map to store time.Time (preserves nanoseconds) - Remove tests/utimensat/08.t from known_failures.txt (all 9 subtests pass) * fix: sync nanosecond fields on all mtime/ctime write paths Ensure MtimeNs/CtimeNs are updated alongside Mtime/Ctime in all code paths: truncate, flush, link, copy_range, metadata flush, and directory touch. * fix: set ctime/ctime_ns in copy_range and metadata flush paths --- test/pjdfstest/known_failures.txt | 4 ---- weed/filer/entry_codec.go | 8 +++++-- weed/mount/weedfs.go | 4 ++-- weed/mount/weedfs_attr.go | 32 +++++++++++++++++++++------- weed/mount/weedfs_file_copy_range.go | 6 +++++- weed/mount/weedfs_file_mkrm.go | 8 ++++--- weed/mount/weedfs_file_sync.go | 8 ++++--- weed/mount/weedfs_link.go | 8 ++++--- weed/mount/weedfs_metadata_flush.go | 6 +++++- weed/pb/filer.proto | 2 ++ weed/pb/filer_pb/filer.pb.go | 24 ++++++++++++++++++--- 11 files changed, 80 insertions(+), 30 deletions(-) diff --git a/test/pjdfstest/known_failures.txt b/test/pjdfstest/known_failures.txt index e6459933f4..41f59bab7c 100644 --- a/test/pjdfstest/known_failures.txt +++ b/test/pjdfstest/known_failures.txt @@ -58,7 +58,3 @@ tests/rename/21.t # entries have been unlinked. tests/unlink/14.t -# ── Nanosecond timestamp precision ───────────────────────────────────── -# The protobuf schema stores timestamps as int64 seconds. Nanosecond -# precision would require adding new fields. -tests/utimensat/08.t diff --git a/weed/filer/entry_codec.go b/weed/filer/entry_codec.go index bff56c008b..0ba82e8959 100644 --- a/weed/filer/entry_codec.go +++ b/weed/filer/entry_codec.go @@ -82,7 +82,9 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes { return &filer_pb.FuseAttributes{ Crtime: entry.Attr.Crtime.Unix(), Mtime: entry.Attr.Mtime.Unix(), + MtimeNs: int32(entry.Attr.Mtime.Nanosecond()), Ctime: entry.Attr.Ctime.Unix(), + CtimeNs: int32(entry.Attr.Ctime.Nanosecond()), FileMode: uint32(entry.Attr.Mode), Uid: entry.Uid, Gid: entry.Gid, @@ -106,7 +108,9 @@ func EntryAttributeToExistingPb(entry *Entry, attr *filer_pb.FuseAttributes) { } attr.Crtime = entry.Attr.Crtime.Unix() attr.Mtime = entry.Attr.Mtime.Unix() + attr.MtimeNs = int32(entry.Attr.Mtime.Nanosecond()) attr.Ctime = entry.Attr.Ctime.Unix() + attr.CtimeNs = int32(entry.Attr.Ctime.Nanosecond()) attr.FileMode = uint32(entry.Attr.Mode) attr.Uid = entry.Uid attr.Gid = entry.Gid @@ -130,9 +134,9 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr { } t.Crtime = time.Unix(attr.Crtime, 0) - t.Mtime = time.Unix(attr.Mtime, 0) + t.Mtime = time.Unix(attr.Mtime, int64(attr.MtimeNs)) if attr.Ctime != 0 { - t.Ctime = time.Unix(attr.Ctime, 0) + t.Ctime = time.Unix(attr.Ctime, int64(attr.CtimeNs)) } else { t.Ctime = t.Mtime } diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index 2f6dd07d17..992e70f80e 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -124,7 +124,7 @@ type WFS struct { refreshMu sync.Mutex refreshingDirs map[util.FullPath]struct{} atimeMu sync.Mutex - atimeMap map[uint64]int64 // inode -> atime (unix seconds), in-memory only, bounded + atimeMap map[uint64]time.Time // inode -> atime, in-memory only, bounded dirHotWindow time.Duration dirHotThreshold int dirIdleEvict time.Duration @@ -204,7 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { fhLockTable: util.NewLockTable[FileHandleId](), posixLocks: NewPosixLockTable(), refreshingDirs: make(map[util.FullPath]struct{}), - atimeMap: make(map[uint64]int64, 8192), + atimeMap: make(map[uint64]time.Time, 8192), dirHotWindow: dirHotWindow, dirHotThreshold: dirHotThreshold, dirIdleEvict: dirIdleEvict, diff --git a/weed/mount/weedfs_attr.go b/weed/mount/weedfs_attr.go index 9157df8e17..bca4cb45be 100644 --- a/weed/mount/weedfs_attr.go +++ b/weed/mount/weedfs_attr.go @@ -92,7 +92,9 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse fh.entryChunkGroup.SetChunks(chunks) } } - entry.Attributes.Mtime = time.Now().Unix() + truncNow := time.Now() + entry.Attributes.Mtime = truncNow.Unix() + entry.Attributes.MtimeNs = int32(truncNow.Nanosecond()) entry.Attributes.FileSize = size } @@ -133,15 +135,18 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse } if atime, ok := input.GetATime(); ok { - wfs.setAtime(input.NodeId, atime.Unix()) + wfs.setAtime(input.NodeId, atime) } if mtime, ok := input.GetMTime(); ok { entry.Attributes.Mtime = mtime.Unix() + entry.Attributes.MtimeNs = int32(mtime.Nanosecond()) } // POSIX: update ctime on any metadata change. - entry.Attributes.Ctime = time.Now().Unix() + now := time.Now() + entry.Attributes.Ctime = now.Unix() + entry.Attributes.CtimeNs = int32(now.Nanosecond()) out.AttrValid = 1 size, includeSize := input.GetSize() @@ -191,12 +196,16 @@ func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.E } out.Blocks = (out.Size + blockSize - 1) / blockSize out.Mtime = uint64(entry.Attributes.Mtime) + out.Mtimensec = uint32(entry.Attributes.MtimeNs) if entry.Attributes.Ctime != 0 { out.Ctime = uint64(entry.Attributes.Ctime) + out.Ctimensec = uint32(entry.Attributes.CtimeNs) } else { out.Ctime = uint64(entry.Attributes.Mtime) + out.Ctimensec = uint32(entry.Attributes.MtimeNs) } out.Atime = uint64(entry.Attributes.Mtime) + out.Atimensec = uint32(entry.Attributes.MtimeNs) // In-memory atime overlay is applied by the caller via applyInMemoryAtime. out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode)) if entry.HardLinkCounter > 0 { @@ -218,11 +227,15 @@ func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.E out.Blocks = (out.Size + blockSize - 1) / blockSize setBlksize(out, blockSize) out.Atime = uint64(entry.Attr.Mtime.Unix()) + out.Atimensec = uint32(entry.Attr.Mtime.Nanosecond()) out.Mtime = uint64(entry.Attr.Mtime.Unix()) + out.Mtimensec = uint32(entry.Attr.Mtime.Nanosecond()) if !entry.Attr.Ctime.IsZero() { out.Ctime = uint64(entry.Attr.Ctime.Unix()) + out.Ctimensec = uint32(entry.Attr.Ctime.Nanosecond()) } else { out.Ctime = uint64(entry.Attr.Mtime.Unix()) + out.Ctimensec = uint32(entry.Attr.Mtime.Nanosecond()) } out.Mode = toSyscallMode(entry.Attr.Mode) if entry.HardLinkCounter > 0 { @@ -258,9 +271,11 @@ func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) { if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil { return } - now := time.Now().Unix() - dirEntry.Attributes.Mtime = now - dirEntry.Attributes.Ctime = now + now := time.Now() + dirEntry.Attributes.Mtime = now.Unix() + dirEntry.Attributes.MtimeNs = int32(now.Nanosecond()) + dirEntry.Attributes.Ctime = now.Unix() + dirEntry.Attributes.CtimeNs = int32(now.Nanosecond()) wfs.saveEntry(dirPath, dirEntry) } @@ -268,7 +283,7 @@ const atimeMapMaxSize = 8192 // setAtime stores an in-memory atime for an inode. The map is bounded; // when full, a random entry is evicted. -func (wfs *WFS) setAtime(inode uint64, t int64) { +func (wfs *WFS) setAtime(inode uint64, t time.Time) { wfs.atimeMu.Lock() defer wfs.atimeMu.Unlock() if len(wfs.atimeMap) >= atimeMapMaxSize { @@ -285,7 +300,8 @@ func (wfs *WFS) setAtime(inode uint64, t int64) { func (wfs *WFS) applyInMemoryAtime(out *fuse.Attr, inode uint64) { wfs.atimeMu.Lock() if t, ok := wfs.atimeMap[inode]; ok { - out.Atime = uint64(t) + out.Atime = uint64(t.Unix()) + out.Atimensec = uint32(t.Nanosecond()) } wfs.atimeMu.Unlock() } diff --git a/weed/mount/weedfs_file_copy_range.go b/weed/mount/weedfs_file_copy_range.go index 1b7b8e407e..f77330a7bc 100644 --- a/weed/mount/weedfs_file_copy_range.go +++ b/weed/mount/weedfs_file_copy_range.go @@ -288,8 +288,12 @@ func synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut *FileHandle, sou } } + copyNow := time.Now() localEntry.Attributes.FileSize = uint64(sourceSize) - localEntry.Attributes.Mtime = time.Now().Unix() + localEntry.Attributes.Mtime = copyNow.Unix() + localEntry.Attributes.MtimeNs = int32(copyNow.Nanosecond()) + localEntry.Attributes.Ctime = copyNow.Unix() + localEntry.Attributes.CtimeNs = int32(copyNow.Nanosecond()) return localEntry } diff --git a/weed/mount/weedfs_file_mkrm.go b/weed/mount/weedfs_file_mkrm.go index d3636af801..0dc69b617b 100644 --- a/weed/mount/weedfs_file_mkrm.go +++ b/weed/mount/weedfs_file_mkrm.go @@ -363,9 +363,11 @@ func (wfs *WFS) truncateEntry(entryFullPath util.FullPath, entry *filer_pb.Entry entry.Content = nil entry.Chunks = nil entry.Attributes.FileSize = 0 - now := time.Now().Unix() - entry.Attributes.Mtime = now - entry.Attributes.Ctime = now + truncNow := time.Now() + entry.Attributes.Mtime = truncNow.Unix() + entry.Attributes.MtimeNs = int32(truncNow.Nanosecond()) + entry.Attributes.Ctime = truncNow.Unix() + entry.Attributes.CtimeNs = int32(truncNow.Nanosecond()) if code := wfs.saveEntry(entryFullPath, entry); code != fuse.OK { return code diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go index 0dcf0e87c6..88c88ea97b 100644 --- a/weed/mount/weedfs_file_sync.go +++ b/weed/mount/weedfs_file_sync.go @@ -189,9 +189,11 @@ func (wfs *WFS) flushMetadataToFiler(fh *FileHandle, dir, name string, uid, gid if entry.Attributes.Gid == 0 { entry.Attributes.Gid = gid } - now := time.Now().Unix() - entry.Attributes.Mtime = now - entry.Attributes.Ctime = now + flushNow := time.Now() + entry.Attributes.Mtime = flushNow.Unix() + entry.Attributes.MtimeNs = int32(flushNow.Nanosecond()) + entry.Attributes.Ctime = flushNow.Unix() + entry.Attributes.CtimeNs = int32(flushNow.Nanosecond()) } glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks())) diff --git a/weed/mount/weedfs_link.go b/weed/mount/weedfs_link.go index b4f399fcca..092bea3e45 100644 --- a/weed/mount/weedfs_link.go +++ b/weed/mount/weedfs_link.go @@ -72,9 +72,11 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out * } // CreateLink 1.2 : update new file to hardlink mode - now := time.Now().Unix() - oldEntry.Attributes.Mtime = now - oldEntry.Attributes.Ctime = now + linkNow := time.Now() + oldEntry.Attributes.Mtime = linkNow.Unix() + oldEntry.Attributes.MtimeNs = int32(linkNow.Nanosecond()) + oldEntry.Attributes.Ctime = linkNow.Unix() + oldEntry.Attributes.CtimeNs = int32(linkNow.Nanosecond()) request := &filer_pb.CreateEntryRequest{ Directory: string(newParentPath), Entry: &filer_pb.Entry{ diff --git a/weed/mount/weedfs_metadata_flush.go b/weed/mount/weedfs_metadata_flush.go index 93a25e6c8b..63c8a08473 100644 --- a/weed/mount/weedfs_metadata_flush.go +++ b/weed/mount/weedfs_metadata_flush.go @@ -111,7 +111,11 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error { entry.Name = name if entry.Attributes != nil { - entry.Attributes.Mtime = time.Now().Unix() + metaNow := time.Now() + entry.Attributes.Mtime = metaNow.Unix() + entry.Attributes.MtimeNs = int32(metaNow.Nanosecond()) + entry.Attributes.Ctime = metaNow.Unix() + entry.Attributes.CtimeNs = int32(metaNow.Nanosecond()) } // Get current chunks - these include chunks that have been uploaded diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 0d31a5e0fd..b06d7ffa96 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -198,6 +198,8 @@ message FuseAttributes { uint32 rdev = 16; uint64 inode = 17; int64 ctime = 18; // unix time in seconds, inode change time + int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999) + int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999) } message CreateEntryRequest { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 2f52defad4..44ccacfa47 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -961,7 +961,9 @@ type FuseAttributes struct { Md5 []byte `protobuf:"bytes,14,opt,name=md5,proto3" json:"md5,omitempty"` Rdev uint32 `protobuf:"varint,16,opt,name=rdev,proto3" json:"rdev,omitempty"` Inode uint64 `protobuf:"varint,17,opt,name=inode,proto3" json:"inode,omitempty"` - Ctime int64 `protobuf:"varint,18,opt,name=ctime,proto3" json:"ctime,omitempty"` // unix time in seconds, inode change time + Ctime int64 `protobuf:"varint,18,opt,name=ctime,proto3" json:"ctime,omitempty"` // unix time in seconds, inode change time + MtimeNs int32 `protobuf:"varint,19,opt,name=mtime_ns,json=mtimeNs,proto3" json:"mtime_ns,omitempty"` // nanosecond component of mtime (0-999999999) + CtimeNs int32 `protobuf:"varint,20,opt,name=ctime_ns,json=ctimeNs,proto3" json:"ctime_ns,omitempty"` // nanosecond component of ctime (0-999999999) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1101,6 +1103,20 @@ func (x *FuseAttributes) GetCtime() int64 { return 0 } +func (x *FuseAttributes) GetMtimeNs() int32 { + if x != nil { + return x.MtimeNs + } + return 0 +} + +func (x *FuseAttributes) GetCtimeNs() int32 { + if x != nil { + return x.CtimeNs + } + return 0 +} + type CreateEntryRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` @@ -5144,7 +5160,7 @@ const file_filer_proto_rawDesc = "" + "\x06FileId\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x19\n" + "\bfile_key\x18\x02 \x01(\x04R\afileKey\x12\x16\n" + - "\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xfe\x02\n" + + "\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xb4\x03\n" + "\x0eFuseAttributes\x12\x1b\n" + "\tfile_size\x18\x01 \x01(\x04R\bfileSize\x12\x14\n" + "\x05mtime\x18\x02 \x01(\x03R\x05mtime\x12\x1b\n" + @@ -5162,7 +5178,9 @@ const file_filer_proto_rawDesc = "" + "\x03md5\x18\x0e \x01(\fR\x03md5\x12\x12\n" + "\x04rdev\x18\x10 \x01(\rR\x04rdev\x12\x14\n" + "\x05inode\x18\x11 \x01(\x04R\x05inode\x12\x14\n" + - "\x05ctime\x18\x12 \x01(\x03R\x05ctime\"\x82\x02\n" + + "\x05ctime\x18\x12 \x01(\x03R\x05ctime\x12\x19\n" + + "\bmtime_ns\x18\x13 \x01(\x05R\amtimeNs\x12\x19\n" + + "\bctime_ns\x18\x14 \x01(\x05R\actimeNs\"\x82\x02\n" + "\x12CreateEntryRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" + "\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12\x15\n" +