fix: skip empty folder cleanup for directory markers

Explicitly created directory markers (zero-byte S3 objects with keys
ending in "/") should not be auto-deleted by the empty folder cleaner
even when they have no children. These represent intentional directory
structures created via PutObject.

The distinction between implicit and explicit directories is:
- Implicit dirs: created by filer as parents, Mime="" (IsOlderDir)
- Explicit markers: created via PutObject, Mime set or ExtMimeType set

Added isDirectoryMarker() check before deletion that looks for
ExtMimeType in extended attributes (set only on zero-byte markers).
Results are cached in folderState to avoid repeated store lookups.
Uses fail-closed on transient errors but allows deletion on ErrNotFound
(entry doesn't exist = implicit directory, not a marker).
This commit is contained in:
Chris Lu committed 2026-03-21 13:23:19 -07:00
1 parent 7e1c2c812f
commit d3b8896094
1 file changed
+59 -4
@@ -2,6 +2,7 @@ package empty_folder_cleanup
import (
"context"
"errors"
"strings"
"sync"
"time"
@@ -31,10 +32,11 @@ type FilerOperations interface {
// folderState tracks the state of a folder for empty folder cleanup
type folderState struct {
roughCount int // Cached rough count (up to maxCountCheck)
lastAddTime time.Time // Last time an item was added
lastDelTime time.Time // Last time an item was deleted
lastCheck time.Time // Last time we checked the actual count
roughCount int // Cached rough count (up to maxCountCheck)
lastAddTime time.Time // Last time an item was added
lastDelTime time.Time // Last time an item was deleted
lastCheck time.Time // Last time we checked the actual count
isDirectoryMarker *bool // Cached check for whether this folder is a directory marker (nil = not checked)
}
type bucketCleanupPolicyState struct {
@@ -312,6 +314,30 @@ func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string)
return
}
// Check if folder is a directory marker (explicitly created S3 object)
// Directory markers should not be auto-deleted even if empty
// Use cached value if available to avoid repeated store lookups
efc.mu.Lock()
state := efc.folderCounts[folder]
var isMarker bool
if state != nil && state.isDirectoryMarker != nil {
isMarker = *state.isDirectoryMarker
} else {
efc.mu.Unlock()
isMarker = efc.isDirectoryMarker(ctx, folder)
// Cache the result
efc.mu.Lock()
if state, exists := efc.folderCounts[folder]; exists && state != nil {
state.isDirectoryMarker = &isMarker
}
}
efc.mu.Unlock()
if isMarker {
glog.V(3).Infof("EmptyFolderCleaner: skipping deletion of directory marker %s (triggered by %s), it was explicitly created", folder, triggeredBy)
return
}
// Delete the empty folder
glog.Infof("EmptyFolderCleaner: deleting empty folder %s (triggered by %s)", folder, triggeredBy)
if err := efc.deleteFolder(ctx, folder); err != nil {
@@ -339,6 +365,35 @@ func (efc *EmptyFolderCleaner) deleteFolder(ctx context.Context, folder string)
return efc.filer.DeleteEntryMetaAndData(ctx, util.FullPath(folder), false, false, false, false, nil, 0)
}
// isDirectoryMarker checks if a folder is a directory marker (explicitly created S3 directory key object)
// Directory markers have a MIME type set in extended attributes (ExtMimeType).
// Returns true (fail-closed) on transient errors to prevent accidental deletion,
// but returns false for ErrNotFound (entry doesn't exist, e.g. implicit directories).
func (efc *EmptyFolderCleaner) isDirectoryMarker(ctx context.Context, folder string) bool {
attrs, err := efc.filer.GetEntryAttributes(ctx, util.FullPath(folder))
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
// Entry doesn't exist in filer (implicit directory) — not a directory marker
return false
}
// Fail-closed on transient errors: assume it's a directory marker
// to prevent accidental deletion due to temporary attribute read failures.
glog.V(2).Infof("EmptyFolderCleaner: error reading attributes for %s, skipping deletion (fail-closed): %v", folder, err)
return true
}
if attrs == nil {
return false
}
// Check if MIME type is set in extended attributes (where we store it for directory markers)
if _, hasMime := attrs[s3_constants.ExtMimeType]; hasMime {
return true
}
return false
}
func (efc *EmptyFolderCleaner) getBucketCleanupPolicy(ctx context.Context, folder string) (bucketPath string, autoRemove bool, source string, attrValue string, err error) {
bucketPath, ok := util.ExtractBucketPath(efc.bucketPath, folder, true)
if !ok {