package filer import ( "context" "testing" "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/remote_storage" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func newMountedTestFiler(t *testing.T, storageType string, stub remote_storage.RemoteStorageClient, listingTtlSeconds int32) (*Filer, *stubFilerStore) { t.Helper() if stub != nil { t.Cleanup(registerStubMaker(t, storageType, stub)) } conf := &remote_pb.RemoteConf{Name: "tombstonestore", Type: storageType} rs := NewFilerRemoteStorage() rs.storageNameToConf[conf.Name] = conf rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ Name: "tombstonestore", Bucket: "mybucket", Path: "/", ListingCacheTtlSeconds: listingTtlSeconds, }) store := newStubFilerStore() return newTestFiler(t, store, rs), store } func putRemoteSyncOffset(t *testing.T, store *stubFilerStore, dir string, offsetTsNs int64) { t.Helper() buf := make([]byte, 8) util.Uint64toBytes(buf, uint64(offsetTsNs)) require.NoError(t, store.KvPut(context.Background(), remote_storage.SyncOffsetKey(dir), buf)) } func TestRemoteDeletionTombstones_BlocksAndReleases(t *testing.T) { tombs := newRemoteDeletionTombstones() tombs.addFromEvent("/m/a.txt", false, 100, 100) _, ts, _ := tombs.blockedSince("/m/a.txt") assert.Equal(t, int64(100), ts) _, ts, _ = tombs.blockedSince("/m/b.txt") assert.Zero(t, ts) // a delete of a directory blocks its whole subtree tombs.addFromEvent("/m/dir", true, 200, 200) _, ts, _ = tombs.blockedSince("/m/dir") assert.Equal(t, int64(200), ts) _, ts, _ = tombs.blockedSince("/m/dir/deep/x.txt") assert.Equal(t, int64(200), ts) _, ts, _ = tombs.blockedSince("/m/dirx/y.txt") assert.Zero(t, ts) // a rewrite at or after the delete lifts only that file's tombstone tombs.clear("/m/a.txt", 100) _, ts, _ = tombs.blockedSince("/m/a.txt") assert.Zero(t, ts) tombs.addFromEvent("/m/a.txt", false, 300, 300) tombs.clear("/m/a.txt", 250) _, ts, _ = tombs.blockedSince("/m/a.txt") assert.Equal(t, int64(300), ts, "older create must not lift newer delete") tombs.releaseThrough("/m/dir/deep/x.txt", 200) _, ts, _ = tombs.blockedSince("/m/dir/deep/x.txt") assert.Zero(t, ts) _, ts, _ = tombs.blockedSince("/m/dir") assert.Zero(t, ts) } func TestRemoteDeletionTombstones_AncestorSubsumesAndCovers(t *testing.T) { tombs := newRemoteDeletionTombstones() // a child tombstone recorded before its ancestor is dropped once the // ancestor's newer delete covers the whole subtree tombs.addFromEvent("/m/dir/a.txt", false, 100, 100) tombs.addFromEvent("/m/dir", true, 200, 200) _, ts, _ := tombs.blockedSince("/m/dir") assert.Equal(t, int64(200), ts) _, ts, _ = tombs.blockedSince("/m/dir/a.txt") assert.Equal(t, int64(200), ts) _, exists := tombs.files["/m/dir/a.txt"] assert.False(t, exists, "descendant tombstone is subsumed by the ancestor") // adds under the covered subtree are skipped while the ancestor stands tombs.addFromEvent("/m/dir/b.txt", false, 150, 150) _, exists = tombs.files["/m/dir/b.txt"] assert.False(t, exists) // ...but a child deleted after the ancestor still records its own tombstone tombs.addFromEvent("/m/dir/c.txt", false, 300, 300) _, ts, _ = tombs.blockedSince("/m/dir/c.txt") assert.Equal(t, int64(300), ts) } func TestRemoteDeletionTombstones_PendingIgnoresOffset(t *testing.T) { f, store := newMountedTestFiler(t, "stub_tomb_pending_offset", nil, 0) // recorded before its event lands: a later event may already have moved // the mount's watermark past the local timestamp, so the offset cannot // vouch for this delete yet filePath := "/buckets/mybucket/dir/a.txt" now := time.Now().UnixNano() f.noteRemoteDeletion(util.FullPath(filePath), false, now) putRemoteSyncOffset(t, store, "/buckets/mybucket", now+100) assert.True(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket")) // once the delete event stamps the real timestamp, the watermark releases it f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket/dir", TsNs: now + 50, EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "a.txt"}, }, }) assert.False(t, f.isRemoteDeletionPending(context.Background(), util.FullPath(filePath), "/buckets/mybucket")) } // A delayed delete is recorded at its commit stamp but delivered under a // bumped stream position. The write-back offset moves on stream positions, // so the tombstone must hold until the offset passes the delivery position, // not the commit stamp — otherwise a lazy read resurrects the remote object. func TestRemoteDeletionTombstones_OffsetMustPassDeliveryPosition(t *testing.T) { f, store := newMountedTestFiler(t, "stub_tomb_stream_pos", nil, 0) base := time.Now().UnixNano() f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket/dir", TsNs: base + 2, // delivered stream position MutationTsNs: base, // commit stamp before a delayed append EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "a.txt"}, }, }) filePath := util.FullPath("/buckets/mybucket/dir/a.txt") putRemoteSyncOffset(t, store, "/buckets/mybucket", base+1) assert.True(t, f.isRemoteDeletionPending(context.Background(), filePath, "/buckets/mybucket"), "offset past the commit stamp must not release a delete delivered later") putRemoteSyncOffset(t, store, "/buckets/mybucket", base+2) assert.False(t, f.isRemoteDeletionPending(context.Background(), filePath, "/buckets/mybucket"), "offset at the delivery position releases the tombstone") } func TestRemoteDeletionTombstones_RaisedLocalTombstoneIsPendingAgain(t *testing.T) { tombs := newRemoteDeletionTombstones() tombs.addFromEvent("/m/a.txt", false, 100, 100) _, _, pending := tombs.blockedSince("/m/a.txt") assert.False(t, pending) // a newer local delete restamps the tombstone before its own event // lands, so the write-back offset cannot vouch for it yet tombs.add("/m/a.txt", false, 200) _, _, pending = tombs.blockedSince("/m/a.txt") assert.True(t, pending) // once that delete's event confirms the new timestamp it releases normally tombs.addFromEvent("/m/a.txt", false, 200, 200) _, _, pending = tombs.blockedSince("/m/a.txt") assert.False(t, pending) } func TestMaybeLazyFetchFromRemote_FullTableKeepsTheFetchOff(t *testing.T) { const storageType = "stub_tomb_full" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) remoteDeletionTombstoneLimit = 2 t.Cleanup(func() { remoteDeletionTombstoneLimit = defaultRemoteDeletionTombstoneLimit }) now := time.Now().UnixNano() f.noteRemoteDeletion("/buckets/mybucket/dir/a.txt", false, now) f.noteRemoteDeletion("/buckets/mybucket/dir/b.txt", false, now) assert.False(t, f.remoteTombstones.isFull()) // the third delete does not fit: from here every path may be a pending // delete, so no lookup miss consults the remote f.noteRemoteDeletion("/buckets/mybucket/dir/c.txt", false, now) assert.True(t, f.remoteTombstones.isFull()) entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) // room again once a recorded delete lifts, but the unrecorded delete of // c.txt has no event to confirm it: the fetch stays off until the TTL f.remoteTombstones.drop("/buckets/mybucket/dir/a.txt", now) assert.True(t, f.remoteTombstones.isFull()) entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) remoteDeletionTombstoneTTL = 0 t.Cleanup(func() { remoteDeletionTombstoneTTL = defaultRemoteDeletionTombstoneTTL }) entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) require.NotNil(t, entry) assert.NotZero(t, stub.statCalls) assert.False(t, f.remoteTombstones.isFull()) } func TestMaybeLazyFetchFromRemote_FullTableDrainsOnReads(t *testing.T) { const storageType = "stub_tomb_drain" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, store := newMountedTestFiler(t, storageType, stub, 0) remoteDeletionTombstoneLimit = 2 t.Cleanup(func() { remoteDeletionTombstoneLimit = defaultRemoteDeletionTombstoneLimit }) now := time.Now().UnixNano() tsA, tsB, tsC := now-3*int64(time.Second), now-2*int64(time.Second), now-int64(time.Second) // a confirmed delete on another mount, then two on this one fill the // table; a third on this mount is unrecorded and closes the fetch remoteDeletionTombstoneLimit = 3 f.remoteTombstones.addFromEvent("/buckets/other/dir/z.txt", false, tsA, tsA) f.remoteTombstones.addFromEvent("/buckets/mybucket/dir/a.txt", false, tsA, tsA) f.remoteTombstones.addFromEvent("/buckets/mybucket/dir/b.txt", false, tsB, tsB) f.remoteTombstones.addFromEvent("/buckets/mybucket/dir/c.txt", false, tsC, tsC) assert.True(t, f.remoteTombstones.isFull()) assert.Len(t, f.remoteTombstones.files, 3) // the write-back offset behind the recorded deletes: a read drains // nothing and the fetch stays off putRemoteSyncOffset(t, store, "/buckets/mybucket", tsA-1) entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) assert.Len(t, f.remoteTombstones.files, 3) // the offset past both recorded deletes of this mount: the read drains // them without any mutation; the other mount's tombstone answers to its // own offset and stays; the unrecorded delete is still inside the TTL, // so the fetch stays off putRemoteSyncOffset(t, store, "/buckets/mybucket", tsB) entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) assert.Len(t, f.remoteTombstones.files, 1) _, otherKept := f.remoteTombstones.files["/buckets/other/dir/z.txt"] assert.True(t, otherKept, "another mount's tombstone must not be released by this mount's offset") assert.True(t, f.remoteTombstones.isFull()) // once the unrecorded delete has aged past the TTL the next read lifts // the gate f.remoteTombstones.mu.Lock() f.remoteTombstones.skippedThroughTsNs = now - int64(remoteDeletionTombstoneTTL) - 1 f.remoteTombstones.mu.Unlock() entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/other.txt") require.NoError(t, err) require.NotNil(t, entry) assert.NotZero(t, stub.statCalls) assert.False(t, f.remoteTombstones.isFull()) } func TestMaybeLazyFetchFromRemote_SkipsTombstonedPath(t *testing.T) { const storageType = "stub_tomb_fetch" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) // a delete under the mount tombstones the path; the remote object is // still there until the write-back daemon consumes the delete event filePath := util.FullPath("/buckets/mybucket/dir/a.txt") f.noteRemoteDeletion(filePath, false, time.Now().UnixNano()) entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath) require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls, "the remote object must not even be consulted") } func TestMaybeLazyFetchFromRemote_NewerRemoteMtimeStillBlocked(t *testing.T) { const storageType = "stub_tomb_regen" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: time.Now().Unix() + 60, RemoteSize: 11}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) // a remote object whose mtime postdates the delete is still hidden: the // remote clock cannot distinguish a new generation from the pending // delete's target filePath := util.FullPath("/buckets/mybucket/dir/a.txt") f.noteRemoteDeletion(filePath, false, time.Now().UnixNano()) entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath) require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) } func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) { const storageType = "stub_tomb_release" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, store := newMountedTestFiler(t, storageType, stub, 0) putDirEntry(store, "/buckets/mybucket/dir") filePath := util.FullPath("/buckets/mybucket/dir/a.txt") deleteTsNs := time.Now().UnixNano() f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket/dir", TsNs: deleteTsNs, EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "a.txt"}, }, }) // the daemon's persisted watermark is behind the delete: still blocked putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs-1) entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath) require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) // once the watermark reaches the delete event's own timestamp, the // remote delete has landed and the lookup may consult the remote again putRemoteSyncOffset(t, store, "/buckets/mybucket", deleteTsNs) entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath) require.NoError(t, err) require.NotNil(t, entry) assert.Equal(t, 1, stub.statCalls) } func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) { const storageType = "stub_tomb_delete" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, store := newMountedTestFiler(t, storageType, stub, 0) putDirEntry(store, "/buckets/mybucket/dir") filePath := util.FullPath("/buckets/mybucket/dir/a.txt") store.entries[string(filePath)] = &Entry{ FullPath: filePath, Attr: Attr{ Mtime: time.Unix(1700000000, 0), Crtime: time.Unix(1700000000, 0), Mode: 0644, FileSize: 11, }, } require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), filePath, false, false, false, false, nil, time.Time{})) // the deleted path must not resurrect through the lazy fetch even while // the remote object is still present entry, err := f.FindEntry(context.Background(), filePath) assert.ErrorIs(t, err, filer_pb.ErrNotFound) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) // a peer-observed create at the path lifts the tombstone f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket/dir", TsNs: time.Now().UnixNano(), EventNotification: &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{Name: "a.txt"}, }, }) entry, err = f.maybeLazyFetchFromRemote(context.Background(), filePath) require.NoError(t, err) require.NotNil(t, entry) assert.Equal(t, 1, stub.statCalls) } func TestOnMetadataChangeEvent_MetadataOnlyDeleteSkipsTombstone(t *testing.T) { const storageType = "stub_tomb_metadata_only" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, store := newMountedTestFiler(t, storageType, stub, 0) putDirEntry(store, "/buckets/mybucket/dir") f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket", TsNs: time.Now().UnixNano(), EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{ Name: "dir", IsDirectory: true, Extended: map[string][]byte{ExtKeepRemoteObjectKey: []byte("true")}, }, }, }) // nothing is owed to the remote, so objects under the folder stay readable entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt") require.NoError(t, err) require.NotNil(t, entry) assert.Equal(t, 1, stub.statCalls) } func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) { const storageType = "stub_tomb_peer" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket/dir", TsNs: time.Now().UnixNano(), EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "a.txt"}, }, }) entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) } func TestOnMetadataChangeEvent_PeerDirDeleteTombstonesSubtree(t *testing.T) { const storageType = "stub_tomb_peer_dir" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket", TsNs: time.Now().UnixNano(), EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true}, }, }) entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/deep/a.txt") require.NoError(t, err) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) } func TestMaybeLazyListFromRemote_SkipsTombstonedChild(t *testing.T) { const storageType = "stub_tomb_list" stub := &stubRemoteClient{ listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error { if err := visitFn("/", "deleted.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10}); err != nil { return err } return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 20}) }, } f, store := newMountedTestFiler(t, storageType, stub, 300) f.noteRemoteDeletion("/buckets/mybucket/deleted.txt", false, time.Now().UnixNano()) f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket")) assert.Equal(t, 1, stub.listDirCalls) assert.Nil(t, store.getEntry("/buckets/mybucket/deleted.txt"), "deleted child must not resurrect through a listing") require.NotNil(t, store.getEntry("/buckets/mybucket/fresh.txt"), "other remote objects still list") } func TestMaybeLazyListFromRemote_RecreatedDirStillHidesChildren(t *testing.T) { const storageType = "stub_tomb_recreate" deleteTs := time.Now().UnixNano() stub := &stubRemoteClient{ listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error { if err := visitFn("/", "stale.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) - 10, RemoteSize: 10}); err != nil { return err } return visitFn("/", "fresh.txt", false, &filer_pb.RemoteEntry{RemoteMtime: deleteTs/int64(time.Second) + 10, RemoteSize: 20}) }, } f, store := newMountedTestFiler(t, storageType, stub, 300) // the directory is deleted then recreated; its remote children stay // hidden — old or new mtime alike — until the remote delete lands f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket", TsNs: deleteTs, EventNotification: &filer_pb.EventNotification{ OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true}, }, }) f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{ Directory: "/buckets/mybucket", TsNs: deleteTs + 1, EventNotification: &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true}, }, }) f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir")) assert.Nil(t, store.getEntry("/buckets/mybucket/dir/stale.txt")) assert.Nil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt")) // after the write-back daemon confirms the remote delete, listing merges again f.remoteTombstones.releaseThrough("/buckets/mybucket/dir/stale.txt", deleteTs) delete(store.getEntry("/buckets/mybucket/dir").Extended, xattrRemoteListingSyncedAt) f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket/dir")) assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/stale.txt")) assert.NotNil(t, store.getEntry("/buckets/mybucket/dir/fresh.txt")) } func TestRemoteDeletionRebuildStartTsNs_UsesOldestMountOffset(t *testing.T) { f, store := newMountedTestFiler(t, "stub_rebuild", nil, 0) f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/other", &remote_pb.RemoteStorageLocation{ Name: "tombstonestore", Bucket: "other", Path: "/", }) mounts := f.RemoteStorage.MountedDirectories() require.Len(t, mounts, 2) now := time.Now().UnixNano() putRemoteSyncOffset(t, store, "/buckets/mybucket", now-200) putRemoteSyncOffset(t, store, "/buckets/other", now-300) assert.Equal(t, now-300, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts), "rebuild must replay from the least-synced mount") // a mount whose offset was never written replays only within the TTL require.NoError(t, store.KvDelete(context.Background(), remote_storage.SyncOffsetKey("/buckets/other"))) floor := f.remoteDeletionRebuildStartTsNs(context.Background(), mounts) assert.GreaterOrEqual(t, floor, time.Now().Add(-remoteDeletionTombstoneTTL-time.Second).UnixNano()) // an offset older than the TTL floor is raised to it putRemoteSyncOffset(t, store, "/buckets/other", time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano()) assert.Greater(t, f.remoteDeletionRebuildStartTsNs(context.Background(), mounts), time.Now().Add(-remoteDeletionTombstoneTTL-time.Hour).UnixNano()) } func TestRebuildRemoteDeletionTombstones_EmptyLogReleasesGate(t *testing.T) { f, _ := newMountedTestFiler(t, "stub_rebuild_empty", nil, 0) f.RebuildRemoteDeletionTombstones(context.Background()) done := f.remoteTombstonesDone.Load() require.NotNil(t, done, "the gate must be set synchronously") select { case <-*done: case <-time.After(10 * time.Second): t.Fatal("rebuild never released the gate") } assert.Nil(t, f.remoteTombstonesDone.Load()) _, ts, _ := f.remoteTombstones.blockedSince("/buckets/mybucket/a.txt") assert.Zero(t, ts) } func TestMaybeLazyFetchFromRemote_WaitsForRebuild(t *testing.T) { const storageType = "stub_tomb_pending" stub := &countingRemoteClient{ stubRemoteClient: stubRemoteClient{ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 10}, }, } f, _ := newMountedTestFiler(t, storageType, stub, 0) // an unfinished rebuild blocks the fetch until the context gives up gate := make(chan struct{}) f.remoteTombstonesDone.Store(&gate) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() entry, err := f.maybeLazyFetchFromRemote(ctx, "/buckets/mybucket/a.txt") assert.ErrorIs(t, err, context.DeadlineExceeded) assert.Nil(t, entry) assert.Equal(t, 0, stub.statCalls) // once the rebuild finishes, the fetch proceeds close(gate) entry, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a.txt") require.NoError(t, err) require.NotNil(t, entry) assert.Equal(t, 1, stub.statCalls) }