filer: keep empty folders that are s3tables catalog entries (#11102)

* s3tables: build the catalog attribute keys from one shared prefix

Every attribute the catalog stores on a bucket, namespace, table or view
entry is spelled out with the same literal prefix. Name it once in
s3_constants so code outside the package can recognize a catalog entry
without repeating the string.

Claude-Session: https://claude.ai/code/session_01GfZsc4cyNB2yr6KYLRv9q1

* filer: keep empty folders that are s3tables catalog entries

A namespace, table or view is a directory whose extended attributes are
the catalog record. Its files can live elsewhere - a rename moves only
the catalog pointer and leaves the data at the old path, and a view has
no files at all - so an empty one is still a live entry.

Drop a table, then rename another table onto that name: the drop queues
the old table's folders, the rename recreates the name path, and two
minutes later the cleaner deletes it and cascades into the namespace,
losing a table the catalog still lists.

Claude-Session: https://claude.ai/code/session_01GfZsc4cyNB2yr6KYLRv9q1

* filer: drop a queued cleanup when the folder is created again

A cleanup is queued against the folder that was found empty. If that
folder is deleted and a new one takes its name, the queue entry outlives
the folder it was about and the next pass deletes the replacement. A
drop followed by a rename onto the dropped name does exactly this: the
name path comes back as a live catalog entry two minutes before the
queue is read.

Claude-Session: https://claude.ai/code/session_01GfZsc4cyNB2yr6KYLRv9q1
This commit is contained in:
Chris Lu authored and GitHub committed 2026-09-02 11:49:08 -07:00
1 parent 23a6b8feb5
commit 86761cc7d5
4 files changed
+122 -8

No files matched your search

@@ -239,6 +239,16 @@ func (efc *EmptyFolderCleaner) OnCreateEvent(directory string, entryName string,
if efc.cleanupQueue.Remove(directory) {
glog.V(3).Infof("EmptyFolderCleaner: cancelled cleanup for %s due to new entry", directory)
}
// A directory that has just been created is a new incarnation, so a cleanup
// queued against the one it replaces would delete it rather than the folder
// that was found empty.
if isDirectory {
recreated := string(util.NewFullPath(directory, entryName))
if efc.cleanupQueue.Remove(recreated) {
glog.V(3).Infof("EmptyFolderCleaner: cancelled cleanup for %s, recreated", recreated)
}
}
}
// cleanupProcessor runs in background and processes the cleanup queue
@@ -529,6 +539,17 @@ func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string)
return
}
// An S3 Tables catalog entry - a namespace, a table whose files a rename left
// behind at the old path, a view that never had any - is the directory itself,
// so its being empty says nothing about whether the catalog still names it.
if extended, err := efc.filer.GetEntryAttributes(ctx, util.FullPath(folder)); err != nil {
glog.V(2).Infof("EmptyFolderCleaner: error reading attributes of %s: %v", folder, err)
return
} else if isCatalogEntry(extended) {
glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), s3tables catalog entry", folder, triggeredBy)
return
}
// Observe it before the delete rather than after. A delete can fail partway and
// still leave the folder gone - the redis stores remove the folder before their
// parent-list member - so a failure return is not proof that it is still there.
@@ -617,6 +638,16 @@ func (efc *EmptyFolderCleaner) getBucketCleanupPolicy(ctx context.Context, folde
return bucketPath, autoRemove, "filer", attrValue, nil
}
// isCatalogEntry reports whether the directory is an s3tables catalog record.
func isCatalogEntry(attrs map[string][]byte) bool {
for key := range attrs {
if strings.HasPrefix(key, s3_constants.ExtS3TablesPrefix) {
return true
}
}
return false
}
func autoRemoveEmptyFoldersEnabled(attrs map[string][]byte) (bool, string) {
if attrs == nil {
return true, "<no_attrs>"