mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-10 05:48:19 +01:00
fix(filer,mount): add nanosecond timestamp precision (#9019)
* fix(filer,mount): add nanosecond timestamp precision Add mtime_ns and ctime_ns fields to the FuseAttributes protobuf message to store the nanosecond component of timestamps (0-999999999). Previously timestamps were truncated to whole seconds. - Update EntryAttributeToPb/PbToEntryAttribute to encode/decode ns - Update setAttrByPbEntry/setAttrByFilerEntry to set Mtimensec/Ctimensec - Update in-memory atime map to store time.Time (preserves nanoseconds) - Remove tests/utimensat/08.t from known_failures.txt (all 9 subtests pass) * fix: sync nanosecond fields on all mtime/ctime write paths Ensure MtimeNs/CtimeNs are updated alongside Mtime/Ctime in all code paths: truncate, flush, link, copy_range, metadata flush, and directory touch. * fix: set ctime/ctime_ns in copy_range and metadata flush paths
This commit is contained in:
1 parent
3f36846642
commit
de5b6f2120
11 files changed
+80
-30
No files matched your search
@@ -58,7 +58,3 @@ tests/rename/21.t
|
||||
# entries have been unlinked.
|
||||
tests/unlink/14.t
|
||||
|
||||
# ── Nanosecond timestamp precision ─────────────────────────────────────
|
||||
# The protobuf schema stores timestamps as int64 seconds. Nanosecond
|
||||
# precision would require adding new fields.
|
||||
tests/utimensat/08.t
|
||||
@@ -82,7 +82,9 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
|
||||
return &filer_pb.FuseAttributes{
|
||||
Crtime: entry.Attr.Crtime.Unix(),
|
||||
Mtime: entry.Attr.Mtime.Unix(),
|
||||
MtimeNs: int32(entry.Attr.Mtime.Nanosecond()),
|
||||
Ctime: entry.Attr.Ctime.Unix(),
|
||||
CtimeNs: int32(entry.Attr.Ctime.Nanosecond()),
|
||||
FileMode: uint32(entry.Attr.Mode),
|
||||
Uid: entry.Uid,
|
||||
Gid: entry.Gid,
|
||||
@@ -106,7 +108,9 @@ func EntryAttributeToExistingPb(entry *Entry, attr *filer_pb.FuseAttributes) {
|
||||
}
|
||||
attr.Crtime = entry.Attr.Crtime.Unix()
|
||||
attr.Mtime = entry.Attr.Mtime.Unix()
|
||||
attr.MtimeNs = int32(entry.Attr.Mtime.Nanosecond())
|
||||
attr.Ctime = entry.Attr.Ctime.Unix()
|
||||
attr.CtimeNs = int32(entry.Attr.Ctime.Nanosecond())
|
||||
attr.FileMode = uint32(entry.Attr.Mode)
|
||||
attr.Uid = entry.Uid
|
||||
attr.Gid = entry.Gid
|
||||
@@ -130,9 +134,9 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
|
||||
}
|
||||
|
||||
t.Crtime = time.Unix(attr.Crtime, 0)
|
||||
t.Mtime = time.Unix(attr.Mtime, 0)
|
||||
t.Mtime = time.Unix(attr.Mtime, int64(attr.MtimeNs))
|
||||
if attr.Ctime != 0 {
|
||||
t.Ctime = time.Unix(attr.Ctime, 0)
|
||||
t.Ctime = time.Unix(attr.Ctime, int64(attr.CtimeNs))
|
||||
} else {
|
||||
t.Ctime = t.Mtime
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ type WFS struct {
|
||||
refreshMu sync.Mutex
|
||||
refreshingDirs map[util.FullPath]struct{}
|
||||
atimeMu sync.Mutex
|
||||
atimeMap map[uint64]int64 // inode -> atime (unix seconds), in-memory only, bounded
|
||||
atimeMap map[uint64]time.Time // inode -> atime, in-memory only, bounded
|
||||
dirHotWindow time.Duration
|
||||
dirHotThreshold int
|
||||
dirIdleEvict time.Duration
|
||||
@@ -204,7 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
|
||||
fhLockTable: util.NewLockTable[FileHandleId](),
|
||||
posixLocks: NewPosixLockTable(),
|
||||
refreshingDirs: make(map[util.FullPath]struct{}),
|
||||
atimeMap: make(map[uint64]int64, 8192),
|
||||
atimeMap: make(map[uint64]time.Time, 8192),
|
||||
dirHotWindow: dirHotWindow,
|
||||
dirHotThreshold: dirHotThreshold,
|
||||
dirIdleEvict: dirIdleEvict,
|
||||
|
||||
@@ -92,7 +92,9 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
|
||||
fh.entryChunkGroup.SetChunks(chunks)
|
||||
}
|
||||
}
|
||||
entry.Attributes.Mtime = time.Now().Unix()
|
||||
truncNow := time.Now()
|
||||
entry.Attributes.Mtime = truncNow.Unix()
|
||||
entry.Attributes.MtimeNs = int32(truncNow.Nanosecond())
|
||||
entry.Attributes.FileSize = size
|
||||
|
||||
}
|
||||
@@ -133,15 +135,18 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
|
||||
}
|
||||
|
||||
if atime, ok := input.GetATime(); ok {
|
||||
wfs.setAtime(input.NodeId, atime.Unix())
|
||||
wfs.setAtime(input.NodeId, atime)
|
||||
}
|
||||
|
||||
if mtime, ok := input.GetMTime(); ok {
|
||||
entry.Attributes.Mtime = mtime.Unix()
|
||||
entry.Attributes.MtimeNs = int32(mtime.Nanosecond())
|
||||
}
|
||||
|
||||
// POSIX: update ctime on any metadata change.
|
||||
entry.Attributes.Ctime = time.Now().Unix()
|
||||
now := time.Now()
|
||||
entry.Attributes.Ctime = now.Unix()
|
||||
entry.Attributes.CtimeNs = int32(now.Nanosecond())
|
||||
|
||||
out.AttrValid = 1
|
||||
size, includeSize := input.GetSize()
|
||||
@@ -191,12 +196,16 @@ func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.E
|
||||
}
|
||||
out.Blocks = (out.Size + blockSize - 1) / blockSize
|
||||
out.Mtime = uint64(entry.Attributes.Mtime)
|
||||
out.Mtimensec = uint32(entry.Attributes.MtimeNs)
|
||||
if entry.Attributes.Ctime != 0 {
|
||||
out.Ctime = uint64(entry.Attributes.Ctime)
|
||||
out.Ctimensec = uint32(entry.Attributes.CtimeNs)
|
||||
} else {
|
||||
out.Ctime = uint64(entry.Attributes.Mtime)
|
||||
out.Ctimensec = uint32(entry.Attributes.MtimeNs)
|
||||
}
|
||||
out.Atime = uint64(entry.Attributes.Mtime)
|
||||
out.Atimensec = uint32(entry.Attributes.MtimeNs)
|
||||
// In-memory atime overlay is applied by the caller via applyInMemoryAtime.
|
||||
out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
|
||||
if entry.HardLinkCounter > 0 {
|
||||
@@ -218,11 +227,15 @@ func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.E
|
||||
out.Blocks = (out.Size + blockSize - 1) / blockSize
|
||||
setBlksize(out, blockSize)
|
||||
out.Atime = uint64(entry.Attr.Mtime.Unix())
|
||||
out.Atimensec = uint32(entry.Attr.Mtime.Nanosecond())
|
||||
out.Mtime = uint64(entry.Attr.Mtime.Unix())
|
||||
out.Mtimensec = uint32(entry.Attr.Mtime.Nanosecond())
|
||||
if !entry.Attr.Ctime.IsZero() {
|
||||
out.Ctime = uint64(entry.Attr.Ctime.Unix())
|
||||
out.Ctimensec = uint32(entry.Attr.Ctime.Nanosecond())
|
||||
} else {
|
||||
out.Ctime = uint64(entry.Attr.Mtime.Unix())
|
||||
out.Ctimensec = uint32(entry.Attr.Mtime.Nanosecond())
|
||||
}
|
||||
out.Mode = toSyscallMode(entry.Attr.Mode)
|
||||
if entry.HardLinkCounter > 0 {
|
||||
@@ -258,9 +271,11 @@ func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) {
|
||||
if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
dirEntry.Attributes.Mtime = now
|
||||
dirEntry.Attributes.Ctime = now
|
||||
now := time.Now()
|
||||
dirEntry.Attributes.Mtime = now.Unix()
|
||||
dirEntry.Attributes.MtimeNs = int32(now.Nanosecond())
|
||||
dirEntry.Attributes.Ctime = now.Unix()
|
||||
dirEntry.Attributes.CtimeNs = int32(now.Nanosecond())
|
||||
wfs.saveEntry(dirPath, dirEntry)
|
||||
}
|
||||
|
||||
@@ -268,7 +283,7 @@ const atimeMapMaxSize = 8192
|
||||
|
||||
// setAtime stores an in-memory atime for an inode. The map is bounded;
|
||||
// when full, a random entry is evicted.
|
||||
func (wfs *WFS) setAtime(inode uint64, t int64) {
|
||||
func (wfs *WFS) setAtime(inode uint64, t time.Time) {
|
||||
wfs.atimeMu.Lock()
|
||||
defer wfs.atimeMu.Unlock()
|
||||
if len(wfs.atimeMap) >= atimeMapMaxSize {
|
||||
@@ -285,7 +300,8 @@ func (wfs *WFS) setAtime(inode uint64, t int64) {
|
||||
func (wfs *WFS) applyInMemoryAtime(out *fuse.Attr, inode uint64) {
|
||||
wfs.atimeMu.Lock()
|
||||
if t, ok := wfs.atimeMap[inode]; ok {
|
||||
out.Atime = uint64(t)
|
||||
out.Atime = uint64(t.Unix())
|
||||
out.Atimensec = uint32(t.Nanosecond())
|
||||
}
|
||||
wfs.atimeMu.Unlock()
|
||||
}
|
||||
|
||||
@@ -288,8 +288,12 @@ func synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut *FileHandle, sou
|
||||
}
|
||||
}
|
||||
|
||||
copyNow := time.Now()
|
||||
localEntry.Attributes.FileSize = uint64(sourceSize)
|
||||
localEntry.Attributes.Mtime = time.Now().Unix()
|
||||
localEntry.Attributes.Mtime = copyNow.Unix()
|
||||
localEntry.Attributes.MtimeNs = int32(copyNow.Nanosecond())
|
||||
localEntry.Attributes.Ctime = copyNow.Unix()
|
||||
localEntry.Attributes.CtimeNs = int32(copyNow.Nanosecond())
|
||||
return localEntry
|
||||
}
|
||||
|
||||
|
||||
@@ -363,9 +363,11 @@ func (wfs *WFS) truncateEntry(entryFullPath util.FullPath, entry *filer_pb.Entry
|
||||
entry.Content = nil
|
||||
entry.Chunks = nil
|
||||
entry.Attributes.FileSize = 0
|
||||
now := time.Now().Unix()
|
||||
entry.Attributes.Mtime = now
|
||||
entry.Attributes.Ctime = now
|
||||
truncNow := time.Now()
|
||||
entry.Attributes.Mtime = truncNow.Unix()
|
||||
entry.Attributes.MtimeNs = int32(truncNow.Nanosecond())
|
||||
entry.Attributes.Ctime = truncNow.Unix()
|
||||
entry.Attributes.CtimeNs = int32(truncNow.Nanosecond())
|
||||
|
||||
if code := wfs.saveEntry(entryFullPath, entry); code != fuse.OK {
|
||||
return code
|
||||
|
||||
@@ -189,9 +189,11 @@ func (wfs *WFS) flushMetadataToFiler(fh *FileHandle, dir, name string, uid, gid
|
||||
if entry.Attributes.Gid == 0 {
|
||||
entry.Attributes.Gid = gid
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
entry.Attributes.Mtime = now
|
||||
entry.Attributes.Ctime = now
|
||||
flushNow := time.Now()
|
||||
entry.Attributes.Mtime = flushNow.Unix()
|
||||
entry.Attributes.MtimeNs = int32(flushNow.Nanosecond())
|
||||
entry.Attributes.Ctime = flushNow.Unix()
|
||||
entry.Attributes.CtimeNs = int32(flushNow.Nanosecond())
|
||||
}
|
||||
|
||||
glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks()))
|
||||
|
||||
@@ -72,9 +72,11 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
}
|
||||
|
||||
// CreateLink 1.2 : update new file to hardlink mode
|
||||
now := time.Now().Unix()
|
||||
oldEntry.Attributes.Mtime = now
|
||||
oldEntry.Attributes.Ctime = now
|
||||
linkNow := time.Now()
|
||||
oldEntry.Attributes.Mtime = linkNow.Unix()
|
||||
oldEntry.Attributes.MtimeNs = int32(linkNow.Nanosecond())
|
||||
oldEntry.Attributes.Ctime = linkNow.Unix()
|
||||
oldEntry.Attributes.CtimeNs = int32(linkNow.Nanosecond())
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: string(newParentPath),
|
||||
Entry: &filer_pb.Entry{
|
||||
|
||||
@@ -111,7 +111,11 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error {
|
||||
entry.Name = name
|
||||
|
||||
if entry.Attributes != nil {
|
||||
entry.Attributes.Mtime = time.Now().Unix()
|
||||
metaNow := time.Now()
|
||||
entry.Attributes.Mtime = metaNow.Unix()
|
||||
entry.Attributes.MtimeNs = int32(metaNow.Nanosecond())
|
||||
entry.Attributes.Ctime = metaNow.Unix()
|
||||
entry.Attributes.CtimeNs = int32(metaNow.Nanosecond())
|
||||
}
|
||||
|
||||
// Get current chunks - these include chunks that have been uploaded
|
||||
|
||||
@@ -198,6 +198,8 @@ message FuseAttributes {
|
||||
uint32 rdev = 16;
|
||||
uint64 inode = 17;
|
||||
int64 ctime = 18; // unix time in seconds, inode change time
|
||||
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
|
||||
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
|
||||
}
|
||||
|
||||
message CreateEntryRequest {
|
||||
|
||||
@@ -961,7 +961,9 @@ type FuseAttributes struct {
|
||||
Md5 []byte `protobuf:"bytes,14,opt,name=md5,proto3" json:"md5,omitempty"`
|
||||
Rdev uint32 `protobuf:"varint,16,opt,name=rdev,proto3" json:"rdev,omitempty"`
|
||||
Inode uint64 `protobuf:"varint,17,opt,name=inode,proto3" json:"inode,omitempty"`
|
||||
Ctime int64 `protobuf:"varint,18,opt,name=ctime,proto3" json:"ctime,omitempty"` // unix time in seconds, inode change time
|
||||
Ctime int64 `protobuf:"varint,18,opt,name=ctime,proto3" json:"ctime,omitempty"` // unix time in seconds, inode change time
|
||||
MtimeNs int32 `protobuf:"varint,19,opt,name=mtime_ns,json=mtimeNs,proto3" json:"mtime_ns,omitempty"` // nanosecond component of mtime (0-999999999)
|
||||
CtimeNs int32 `protobuf:"varint,20,opt,name=ctime_ns,json=ctimeNs,proto3" json:"ctime_ns,omitempty"` // nanosecond component of ctime (0-999999999)
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1101,6 +1103,20 @@ func (x *FuseAttributes) GetCtime() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FuseAttributes) GetMtimeNs() int32 {
|
||||
if x != nil {
|
||||
return x.MtimeNs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FuseAttributes) GetCtimeNs() int32 {
|
||||
if x != nil {
|
||||
return x.CtimeNs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type CreateEntryRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"`
|
||||
@@ -5144,7 +5160,7 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\x06FileId\x12\x1b\n" +
|
||||
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x19\n" +
|
||||
"\bfile_key\x18\x02 \x01(\x04R\afileKey\x12\x16\n" +
|
||||
"\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xfe\x02\n" +
|
||||
"\x06cookie\x18\x03 \x01(\aR\x06cookie\"\xb4\x03\n" +
|
||||
"\x0eFuseAttributes\x12\x1b\n" +
|
||||
"\tfile_size\x18\x01 \x01(\x04R\bfileSize\x12\x14\n" +
|
||||
"\x05mtime\x18\x02 \x01(\x03R\x05mtime\x12\x1b\n" +
|
||||
@@ -5162,7 +5178,9 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\x03md5\x18\x0e \x01(\fR\x03md5\x12\x12\n" +
|
||||
"\x04rdev\x18\x10 \x01(\rR\x04rdev\x12\x14\n" +
|
||||
"\x05inode\x18\x11 \x01(\x04R\x05inode\x12\x14\n" +
|
||||
"\x05ctime\x18\x12 \x01(\x03R\x05ctime\"\x82\x02\n" +
|
||||
"\x05ctime\x18\x12 \x01(\x03R\x05ctime\x12\x19\n" +
|
||||
"\bmtime_ns\x18\x13 \x01(\x05R\amtimeNs\x12\x19\n" +
|
||||
"\bctime_ns\x18\x14 \x01(\x05R\actimeNs\"\x82\x02\n" +
|
||||
"\x12CreateEntryRequest\x12\x1c\n" +
|
||||
"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" +
|
||||
"\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12\x15\n" +
|
||||
|
||||
Reference in new issue
Block a user