s3: tear down the emptied .versions directory on last-version delete; drain existing residue (#10278)

* s3: routed last-version delete removes the emptied .versions directory

The routed versioned delete (routedDeleteSpecificVersion) repoints the
latest pointer and deletes the version file, but unlike the lock-path
fallback (updateLatestVersionAfterDeletion) it never tears down the
.versions/ directory it just emptied. The residue keeps the key's read
path in the self-heal rescan loop: every GET of the deleted key logs
event=surfaced plus a GetObject error until the background
EmptyFolderCleaner gets to the directory — at least two minutes away on
its delay queue, and possibly never (the queue is in-memory, bounded,
and gated on the bucket's allow-empty-folders policy). Veeam's lock
arbitration probes deleted lock keys continuously, so those windows are
always open and the log spam is chronic.

ObjectMutation DELETE gains remove_empty_parent: after the child delete,
the filer best-effort removes the parent directory in the same locked
transaction. Non-recursive on purpose — a concurrent write that lands a
new version fails the removal instead of being lost with it. The routed
last-version delete sets it on the version-file DELETE, matching the
lock-path fallback's contract.

Claude-Session: https://claude.ai/code/session_014mMYAHXZySkCCUfpRFNtSv

* s3: drain empty .versions residue on read heal and in s3.versions.audit

Directories already stranded by pre-teardown deletes (or dropped from
the EmptyFolderCleaner's bounded in-memory queue) previously re-entered
the self-heal rescan on every GET forever: the heal only cleared the
pointer and nothing ever removed the directory, and s3.versions.audit
counted the state as clean.

When the heal rescan finds no remaining version, remove the directory
outright (non-recursive, so orphan children still block and fall back to
the pointer clear) and log event=healed mode=empty_dir_removed; the next
GET takes the clean not-found path. The audit gains an empty category so
the residue is visible, and -heal removes such directories in bulk.

Claude-Session: https://claude.ai/code/session_014mMYAHXZySkCCUfpRFNtSv
This commit is contained in:
Chris Lu authored and GitHub committed 2026-07-08 18:50:54 -07:00
1 parent 11fd20e2b0
commit a9cfbd8d3a
9 files changed
+191 -35

No files matched your search

+15 -3
View File
@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -391,10 +392,21 @@ func (fs *FilerServer) applyObjectMutation(ctx context.Context, m *filer_pb.Obje
case filer_pb.ObjectMutation_DELETE:
fullpath := util.NewFullPath(m.Directory, m.Name)
err := fs.filer.DeleteEntryMetaAndData(ctx, fullpath, m.IsRecursive, false, m.IsDeleteData, fromOtherCluster, signatures, 0)
if err == filer_pb.ErrNotFound {
return nil
if err != nil && err != filer_pb.ErrNotFound {
return err
}
return err
if m.RemoveEmptyParent {
// A parent that exists only to hold this child (e.g. a .versions/
// directory losing its last version) is torn down in the same locked
// transaction. Best-effort: a non-empty or already-removed parent is
// the expected no-op, and a failed teardown must not fail the
// already-applied delete.
parentErr := fs.filer.DeleteEntryMetaAndData(ctx, util.FullPath(m.Directory), false, false, false, fromOtherCluster, signatures, 0)
if parentErr != nil && parentErr != filer_pb.ErrNotFound && !strings.Contains(parentErr.Error(), filer.MsgFailDelNonEmptyFolder) {
glog.V(1).InfofCtx(ctx, "remove empty parent %s: %v", m.Directory, parentErr)
}
}
return nil
case filer_pb.ObjectMutation_PATCH_EXTENDED:
fullpath := util.NewFullPath(m.Directory, m.Name)
@@ -619,6 +619,86 @@ func TestObjectTransactionVersionDeleteWithWorm(t *testing.T) {
}
}
// Deleting the LAST version tears down the emptied .versions/ directory in the
// same transaction (remove_empty_parent on the DELETE), so a fully-drained key
// leaves no residue to re-trigger the read path's self-heal on every GET. A
// remaining child keeps the directory; a replay after the child is already
// gone still tidies it.
func TestObjectTransactionDeleteRemovesEmptyParent(t *testing.T) {
now := time.Unix(1700000000, 0)
seed := func(withSibling bool) map[string]*filer.Entry {
entries := map[string]*filer.Entry{
"/buckets/b/obj/.versions": {
Attr: filer.Attr{Inode: 2, Mtime: now, Crtime: now, Mode: 0755 | (1 << 31)},
Extended: map[string][]byte{"latestName": []byte("v_a"), "latestVid": []byte("v1")},
},
"/buckets/b/obj/.versions/v_a": {
Attr: filer.Attr{Inode: 10, Mtime: now, Crtime: now, Mode: 0644},
Extended: map[string][]byte{"vid": []byte("v1")},
},
}
if withSibling {
entries["/buckets/b/obj/.versions/orphan"] = &filer.Entry{
Attr: filer.Attr{Inode: 11, Mtime: now, Crtime: now, Mode: 0644},
}
}
return entries
}
// Mirrors routedDeleteSpecificVersion: repoint excluding the dying version,
// then delete it and tidy the parent.
mkReq := func() *filer_pb.ObjectTransactionRequest {
return &filer_pb.ObjectTransactionRequest{
LockKey: "/buckets/b/obj",
Mutations: []*filer_pb.ObjectMutation{
{Type: filer_pb.ObjectMutation_RECOMPUTE_LATEST, Directory: "/buckets/b/obj", Name: ".versions",
Recompute: &filer_pb.Recompute{ScanDir: "/buckets/b/obj/.versions", Descending: true, ExcludeName: "v_a",
NameToKey: "latestName", CopyExtended: map[string]string{"latestVid": "vid"}}},
{Type: filer_pb.ObjectMutation_DELETE, Directory: "/buckets/b/obj/.versions", Name: "v_a", RemoveEmptyParent: true},
},
}
}
// Last version: the emptied directory goes with it.
fs, store := newTxnTestServer(seed(false))
resp, err := fs.ObjectTransaction(context.Background(), mkReq())
if err != nil || resp.Error != "" {
t.Fatalf("txn failed: err=%v resp=%q", err, resp.Error)
}
if _, ok := store.entries["/buckets/b/obj/.versions/v_a"]; ok {
t.Errorf("version should be deleted")
}
if _, ok := store.entries["/buckets/b/obj/.versions"]; ok {
t.Errorf(".versions directory emptied by the delete should be removed")
}
// A remaining child keeps the directory (non-recursive teardown declines).
fs, store = newTxnTestServer(seed(true))
resp, err = fs.ObjectTransaction(context.Background(), mkReq())
if err != nil || resp.Error != "" {
t.Fatalf("txn failed: err=%v resp=%q", err, resp.Error)
}
if _, ok := store.entries["/buckets/b/obj/.versions/orphan"]; !ok {
t.Errorf("sibling must survive the teardown attempt")
}
if _, ok := store.entries["/buckets/b/obj/.versions"]; !ok {
t.Errorf(".versions directory with a remaining child must not be removed")
}
// Replay: the child is already gone, the empty parent is still tidied.
fs, store = newTxnTestServer(map[string]*filer.Entry{
"/buckets/b/obj/.versions": {
Attr: filer.Attr{Inode: 2, Mtime: now, Crtime: now, Mode: 0755 | (1 << 31)},
},
})
resp, err = fs.ObjectTransaction(context.Background(), mkReq())
if err != nil || resp.Error != "" {
t.Fatalf("replay txn failed: err=%v resp=%q", err, resp.Error)
}
if _, ok := store.entries["/buckets/b/obj/.versions"]; ok {
t.Errorf("replayed delete should still remove the empty directory")
}
}
// PATCH_EXTENDED with touch_mtime bumps the entry's Mtime (a metadata-replace
// copy) while merging Extended.
func TestObjectTransactionPatchTouchMtime(t *testing.T) {