filer: support nanosecond timestamps for conditional delete (#11339) (#12071)

* filer: support nanosecond timestamps for conditional delete (#11339)

* shell: fs.verify prune carries the entry mtime in nanoseconds

IfNotModifiedAfter now resolves to sub-second precision on the filer,
so pass the looked-up mtime_ns along with mtime; the re-lookup check
catches same-second rewrites that share the second, and the test mock
mirrors the filer's seconds/nanoseconds cutoff.

* shell: volume.fsck guards empty-directory deletes with mtime nanoseconds

The looked-up mtime_ns joins the seconds in IfNotModifiedAfter so a
directory rewritten within the same second is not dropped; the stub
filer mirrors the filer's scale heuristic and the race fixtures use a
realistic mtime so the nanosecond branch engages.

* filer: cover the seconds guard and equality boundary

Test second-resolution cutoffs on both sides of the entry's second and
the exact-equality nanosecond guard that still deletes.

* filer: compare the nanosecond guard as a time, not UnixNano

UnixNano overflows for mtimes outside 1678-2262, letting a far-future
entry slip past the guard; time.After compares exactly at every epoch.
The seconds branch keeps Unix() so a same-second cutoff still deletes.

* filer_pb: add explicit if_not_modified_after_ns to DeleteEntryRequest

* filer: prefer explicit nanoseconds guard, keep if_not_modified_after as seconds

* shell: send seconds and nanoseconds delete cutoffs, skip unexpressible mtimes

* filer: merge conditional-delete guards into one nanosecond cutoff

DeleteEntryMetaAndData carried both the seconds and nanoseconds guards
even though the precedence between them is fixed, so every caller passed
a dead parameter. The gRPC handler now folds if_not_modified_after into
nanoseconds at the boundary, where (sec+1)*1e9-1 keeps the whole-second
cutoff exact, and the filer takes a single ifNotModifiedAfterNs.

* filer: hold the conditional-delete cutoff as a time, not int64 nanoseconds

A seconds-only guard at or past year 2262 saturated to MaxInt64
nanoseconds (~2262), so a delete whose cutoff matched a far-future mtime
was skipped while the request still reported success, and any entry
between 2262 and the real cutoff survived pruning. The merged cutoff is
now a time.Time, where a whole-second guard maps to the last nanosecond
of its second for any int64 second.

---------

Co-authored-by: user.email <chrislusf@users.noreply.github.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/shell/command_fs_verify.go

@@ -163,8 +163,7 @@ type ItemEntry struct {
 	chunks    []*filer_pb.FileChunk
 	rawChunks []*filer_pb.FileChunk
 	path      util.FullPath
-	mtimeSec  int64
-	md5       []byte
+	attr      *filer_pb.FuseAttributes
 }
 
 // resolveAndVerify expands chunk manifests and verifies the entry's needles.
@@ -220,8 +219,7 @@ func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup)
 					if *c.pruneEntries && hasMissingNeedles {
 						pruned, pruneErr := c.pruneEntry(
 							util.NewFullPath(message.NewParentPath, message.NewEntry.Name),
-							message.NewEntry.Attributes.Mtime,
-							message.NewEntry.Attributes.Md5,
+							message.NewEntry.Attributes,
 							message.NewEntry.Chunks)
 						if pruneErr != nil {
 							fmt.Fprintf(c.writer, "prune %s failed: %v\n", entryPath, pruneErr)
@@ -371,13 +369,28 @@ func (c *commandFsVerify) verifyEntry(path string, chunks []*filer_pb.FileChunk,
 	return itemIsVerifed.Load(), hasMissingNeedlesFlag.Load()
 }
 
+// deleteEntryCutoff splits an entry mtime into the conditional-delete fields:
+// seconds keep older filers enforcing the guard, nanoseconds keep same-second
+// precision where int64 can hold them. ok is false when neither field can
+// express the cutoff and the delete must be skipped rather than sent unguarded.
+func deleteEntryCutoff(mtimeSec int64, mtimeNsec int32) (sec, ns int64, ok bool) {
+	sec = mtimeSec
+	if sec <= 0 {
+		return 0, 0, false
+	}
+	if nsec := int64(mtimeNsec); nsec >= 0 && nsec < 1e9 && sec <= (math.MaxInt64-nsec)/1e9 {
+		ns = sec*1e9 + nsec
+	}
+	return sec, ns, true
+}
+
 // pruneEntry deletes an entry whose needles are gone, unless the entry
 // changed since verification. Three guards protect a concurrent repair:
 // re-lookup with mtime/md5 comparison, an identity check of the chunk set
 // (an append or rewrite in the same second as the original mtime changes the
 // chunks but not the timestamps), and IfNotModifiedAfter on the delete. The
 // entry is only counted as pruned when a follow-up lookup confirms it is gone.
-func (c *commandFsVerify) pruneEntry(path util.FullPath, mtimeSec int64, md5 []byte, expectedChunks []*filer_pb.FileChunk) (pruned bool, err error) {
+func (c *commandFsVerify) pruneEntry(path util.FullPath, attr *filer_pb.FuseAttributes, expectedChunks []*filer_pb.FileChunk) (pruned bool, err error) {
 	dir, name := path.DirAndName()
 	expectedIds := chunkFingerprints(expectedChunks)
 	lookupErr := c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
@@ -392,11 +405,11 @@ func (c *commandFsVerify) pruneEntry(path util.FullPath, mtimeSec int64, md5 []b
 			return err
 		}
 		att := lookupResp.Entry.GetAttributes()
-		if att.GetMtime() != mtimeSec {
+		if att.GetMtime() != attr.GetMtime() || att.GetMtimeNs() != attr.GetMtimeNs() {
 			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
 			return nil
 		}
-		if len(md5) > 0 && !bytes.Equal(att.GetMd5(), md5) {
+		if md5 := attr.GetMd5(); len(md5) > 0 && !bytes.Equal(att.GetMd5(), md5) {
 			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
 			return nil
 		}
@@ -404,10 +417,16 @@ func (c *commandFsVerify) pruneEntry(path util.FullPath, mtimeSec int64, md5 []b
 			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
 			return nil
 		}
+		sec, ns, ok := deleteEntryCutoff(attr.GetMtime(), attr.GetMtimeNs())
+		if !ok {
+			fmt.Fprintf(c.writer, "skip pruning %s: mtime cannot set a delete condition\n", path)
+			return nil
+		}
 		deleteResp, err := client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
-			Directory:          dir,
-			Name:               name,
-			IfNotModifiedAfter: mtimeSec,
+			Directory:            dir,
+			Name:                 name,
+			IfNotModifiedAfter:   sec,
+			IfNotModifiedAfterNs: ns,
 		})
 		if err != nil {
 			return err
@@ -487,8 +506,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errC
 					chunks:    dataChunks,
 					rawChunks: entry.Entry.GetChunks(),
 					path:      util.NewFullPath(entry.Dir, entry.Entry.Name),
-					mtimeSec:  entry.Entry.GetAttributes().GetMtime(),
-					md5:       entry.Entry.GetAttributes().GetMd5(),
+					attr:      entry.Entry.GetAttributes(),
 				}:
 				case <-ctx.Done():
 					return ctx.Err()
@@ -508,7 +526,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errC
 					}
 					fileCount++
 				} else if *c.pruneEntries && hasMissingNeedles {
-					pruned, pruneErr := c.pruneEntry(i.path, i.mtimeSec, i.md5, i.rawChunks)
+					pruned, pruneErr := c.pruneEntry(i.path, i.attr, i.rawChunks)
 					if pruneErr != nil {
 						fmt.Fprintf(c.writer, "prune %s failed: %v\n", itemPath, pruneErr)
 					} else if pruned {

weed/shell/command_fs_verify_test.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -48,10 +50,18 @@ func (s *fsVerifyTestFilerServer) LookupDirectoryEntry(_ context.Context, req *f
 
 func (s *fsVerifyTestFilerServer) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
 	s.deleted = append(s.deleted, req)
-	// mirror the real filer: an entry newer than IfNotModifiedAfter is
+	// mirror the real filer: an entry newer than the cutoff is
 	// silently skipped, no error and no Error field
-	if s.entry != nil && req.IfNotModifiedAfter > 0 && s.entry.Attributes.GetMtime() > req.IfNotModifiedAfter {
-		return &filer_pb.DeleteEntryResponse{}, nil
+	if s.entry != nil {
+		entryNewer := false
+		if req.IfNotModifiedAfterNs > 0 {
+			entryNewer = s.entry.Attributes.GetMtime()*1e9+int64(s.entry.Attributes.GetMtimeNs()) > req.IfNotModifiedAfterNs
+		} else if req.IfNotModifiedAfter > 0 {
+			entryNewer = s.entry.Attributes.GetMtime() > req.IfNotModifiedAfter
+		}
+		if entryNewer {
+			return &filer_pb.DeleteEntryResponse{}, nil
+		}
 	}
 	s.entry = nil
 	return &filer_pb.DeleteEntryResponse{}, nil
@@ -234,13 +244,16 @@ func TestFsVerifyVerifyEntryPrunability(t *testing.T) {
 // Pruning must only delete the entry when it did not change since the
 // verification pass — a re-upload fixes the file and must survive.
 func TestFsVerifyPruneEntryGuards(t *testing.T) {
+	mtimeNs := time.Now().Add(-time.Hour).UnixNano()
+	attr := &filer_pb.FuseAttributes{
+		Mtime:   mtimeNs / 1e9,
+		MtimeNs: int32(mtimeNs % 1e9),
+		Md5:     []byte("md5"),
+	}
 	filer := &fsVerifyTestFilerServer{entry: &filer_pb.Entry{
-		Name: "file",
-		Attributes: &filer_pb.FuseAttributes{
-			Mtime: 100,
-			Md5:   []byte("md5"),
-		},
-		Chunks: []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 42}}},
+		Name:       "file",
+		Attributes: attr,
+		Chunks:     []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 42}}},
 	}}
 	commandEnv, _, cleanup := newFsVerifyTestCommandEnv(t, filer, &fsVerifyTestVolumeServer{})
 	defer cleanup()
@@ -248,27 +261,43 @@ func TestFsVerifyPruneEntryGuards(t *testing.T) {
 
 	chunks := []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 42}}}
 
-	// unchanged entry: deleted
-	pruned, err := c.pruneEntry(util.NewFullPath("/buckets/b", "file"), 100, []byte("md5"), chunks)
+	// unchanged entry: deleted; the request carries the cutoff in seconds
+	// for older filers and nanoseconds for same-second precision
+	pruned, err := c.pruneEntry(util.NewFullPath("/buckets/b", "file"), attr, chunks)
 	if err != nil || !pruned {
 		t.Fatalf("expected prune of unchanged entry, got pruned=%v err=%v", pruned, err)
 	}
+	if len(filer.deleted) != 1 || filer.deleted[0].IfNotModifiedAfter != mtimeNs/1e9 || filer.deleted[0].IfNotModifiedAfterNs != mtimeNs {
+		t.Fatalf("delete sent wrong cutoffs: %+v", filer.deleted)
+	}
 
 	// re-uploaded entry (new mtime/md5): must survive
-	filer.entry = &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{Mtime: 200, Md5: []byte("new")}}
-	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), 100, []byte("md5"), chunks)
+	filer.entry = &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{Mtime: mtimeNs/1e9 + 1, Md5: []byte("new")}}
+	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), attr, chunks)
 	if err != nil || pruned {
 		t.Fatalf("changed entry must not be pruned, got pruned=%v err=%v", pruned, err)
 	}
 
+	// same-second rewrite that only advanced the nanosecond component:
+	// must survive
+	filer.entry = &filer_pb.Entry{
+		Name:       "file",
+		Attributes: &filer_pb.FuseAttributes{Mtime: mtimeNs / 1e9, MtimeNs: int32(mtimeNs%1e9) + 1, Md5: []byte("md5")},
+		Chunks:     []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 42}}},
+	}
+	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), attr, chunks)
+	if err != nil || pruned {
+		t.Fatalf("sub-second rewrite must not be pruned, got pruned=%v err=%v", pruned, err)
+	}
+
 	// same-second rewrite: timestamps identical but the chunk set differs —
 	// the new upload must survive
 	filer.entry = &filer_pb.Entry{
 		Name:       "file",
-		Attributes: &filer_pb.FuseAttributes{Mtime: 100, Md5: []byte("md5")},
+		Attributes: &filer_pb.FuseAttributes{Mtime: mtimeNs / 1e9, MtimeNs: int32(mtimeNs % 1e9), Md5: []byte("md5")},
 		Chunks:     []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 999}}},
 	}
-	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), 100, []byte("md5"), chunks)
+	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), attr, chunks)
 	if err != nil || pruned {
 		t.Fatalf("same-second rewrite must not be pruned, got pruned=%v err=%v", pruned, err)
 	}
@@ -276,22 +305,48 @@ func TestFsVerifyPruneEntryGuards(t *testing.T) {
 	// identical chunk set (order-insensitive): prunes
 	filer.entry = &filer_pb.Entry{
 		Name:       "file",
-		Attributes: &filer_pb.FuseAttributes{Mtime: 100, Md5: []byte("md5")},
+		Attributes: &filer_pb.FuseAttributes{Mtime: mtimeNs / 1e9, MtimeNs: int32(mtimeNs % 1e9), Md5: []byte("md5")},
 		Chunks:     []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 42}}},
 	}
-	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), 100, []byte("md5"), chunks)
+	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), attr, chunks)
 	if err != nil || !pruned {
 		t.Fatalf("expected prune when chunk set matches, got pruned=%v err=%v", pruned, err)
 	}
 
 	// entry already gone: no error, no delete
 	filer.entry = nil
-	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), 100, nil, chunks)
+	pruned, err = c.pruneEntry(util.NewFullPath("/buckets/b", "file"), &filer_pb.FuseAttributes{Mtime: mtimeNs / 1e9, MtimeNs: int32(mtimeNs % 1e9)}, chunks)
 	if err != nil || pruned {
 		t.Fatalf("missing entry must be a no-op, got pruned=%v err=%v", pruned, err)
 	}
 }
 
+func TestDeleteEntryCutoff(t *testing.T) {
+	// a regular mtime fills both fields
+	sec, ns, ok := deleteEntryCutoff(1_761_504_000, 500_000_000)
+	if !ok || sec != 1_761_504_000 || ns != 1_761_504_000_500_000_000 {
+		t.Fatalf("regular mtime: got %d %d %v", sec, ns, ok)
+	}
+	// an mtime in the first seconds after the epoch is unambiguous in both
+	// fields once units are explicit
+	sec, ns, ok = deleteEntryCutoff(100, 0)
+	if !ok || sec != 100 || ns != 100_000_000_000 {
+		t.Fatalf("epoch mtime: got %d %d %v", sec, ns, ok)
+	}
+	// a pre-epoch mtime cannot arm either field — the delete must be skipped
+	if _, _, ok = deleteEntryCutoff(-100, 0); ok {
+		t.Fatal("pre-epoch mtime must report ok=false")
+	}
+	if _, _, ok = deleteEntryCutoff(0, 0); ok {
+		t.Fatal("zero mtime must report ok=false")
+	}
+	// an mtime past the UnixNano range degrades to the seconds field
+	sec, ns, ok = deleteEntryCutoff(math.MaxInt64/1_000_000_000+1, 0)
+	if !ok || ns != 0 || sec != math.MaxInt64/1_000_000_000+1 {
+		t.Fatalf("overflow mtime: got %d %d %v", sec, ns, ok)
+	}
+}
+
 // An unresolvable chunk manifest is an entry-level verification failure even
 // when the raw manifest needle is healthy; raw chunks are still checked so a
 // missing manifest needle can be pruned.

weed/shell/command_volume_fsck.go

@@ -863,7 +863,7 @@ func (c *commandVolumeFsck) purgeEmptyDirectories() {
 		if mtime <= 0 || mtime >= time.Now().Add(-directoryQuietPeriod).Unix() {
 			continue
 		}
-		if err := c.deleteEmptyDirectory(dir, mtime); err != nil {
+		if err := c.deleteEmptyDirectory(dir, entry.Attributes); err != nil {
 			if !errors.Is(err, filer.ErrNonEmptyFolder) {
 				fmt.Fprintf(c.writer, "delete empty directory %s: %v\n", dir, err)
 			}
@@ -873,16 +873,21 @@ func (c *commandVolumeFsck) purgeEmptyDirectories() {
 	}
 }
 
-// deleteEmptyDirectory deletes dir unless it changed since it was looked up at mtime,
-// so a directory promoted to an S3 object meanwhile survives. The delete is not
-// recursive, leaving the filer to reject a directory that is not empty.
-func (c *commandVolumeFsck) deleteEmptyDirectory(dir util.FullPath, mtime int64) error {
+// deleteEmptyDirectory deletes dir unless it changed since it was looked up,
+// so a directory promoted to an S3 object meanwhile survives. The delete is
+// not recursive, leaving the filer to reject a directory that is not empty.
+func (c *commandVolumeFsck) deleteEmptyDirectory(dir util.FullPath, attr *filer_pb.FuseAttributes) error {
 	parent, name := dir.DirAndName()
+	sec, ns, ok := deleteEntryCutoff(attr.GetMtime(), attr.GetMtimeNs())
+	if !ok {
+		return nil
+	}
 	return c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
-			Directory:          parent,
-			Name:               name,
-			IfNotModifiedAfter: mtime,
+			Directory:            parent,
+			Name:                 name,
+			IfNotModifiedAfter:   sec,
+			IfNotModifiedAfterNs: ns,
 		})
 		if err != nil {
 			return err

weed/shell/command_volume_fsck_test.go

@@ -107,7 +107,13 @@ func (s *stubFilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteE
 	if !found {
 		return &filer_pb.DeleteEntryResponse{Error: filer_pb.ErrNotFound.Error()}, nil
 	}
-	if req.IfNotModifiedAfter > 0 && entry.Attributes.GetMtime() > req.IfNotModifiedAfter {
+	entryNewer := false
+	if req.IfNotModifiedAfterNs > 0 {
+		entryNewer = entry.Attributes.GetMtime()*1e9+int64(entry.Attributes.GetMtimeNs()) > req.IfNotModifiedAfterNs
+	} else if req.IfNotModifiedAfter > 0 {
+		entryNewer = entry.Attributes.GetMtime() > req.IfNotModifiedAfter
+	}
+	if entryNewer {
 		return &filer_pb.DeleteEntryResponse{}, nil
 	}
 	for path := range s.entries {
@@ -209,8 +215,8 @@ func TestVolumeFsckPurgeEmptyDirectoriesKeepsFreshDirectory(t *testing.T) {
 func TestVolumeFsckPurgeEmptyDirectoriesKeepsChangedDirectory(t *testing.T) {
 	stub := &stubFilerServer{
 		entries: map[string]*filer_pb.Entry{
-			"/race":     {IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: 100}},
-			"/race/dir": {IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: 100}},
+			"/race":     {IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Add(-time.Hour).Unix()}},
+			"/race/dir": {IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Add(-time.Hour).Unix()}},
 		},
 		writtenAfterLookup: map[string]bool{"/race/dir": true},
 	}