mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 03:43:41 +01:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d5236af36 | ||
|
|
75967b355c | ||
|
|
de470c66fb | ||
|
|
4f90865348 | ||
|
|
15ac3500a3 | ||
|
|
876e98ff15 | ||
|
|
77cc6f3cbc | ||
|
|
9009825fcc | ||
|
|
d1dc0889f3 | ||
|
|
f4f3fa35f6 | ||
|
|
1aa88de0ac | ||
|
|
bdda9d8879 | ||
|
|
5fb41d946e | ||
|
|
57ae6ffce0 | ||
|
|
4dd9cc289d | ||
|
|
6d733cb863 | ||
|
|
bb06389a8c | ||
|
|
c701d57535 |
No files matched your search
+5
-3
@@ -772,8 +772,10 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
|
||||
}
|
||||
entry, err = f.Store.FindEntry(ctx, p)
|
||||
// A directory is deleted here one row at a time, which would strand whatever is
|
||||
// under it, so a TTL an older build left on one is not acted on.
|
||||
if entry != nil && entry.TtlSec > 0 && !entry.IsDirectory() {
|
||||
// under it, so a TTL an older build left on one is not acted on. Retention
|
||||
// outranks expiry the same way it outranks lifecycle rules on S3: a locked
|
||||
// entry stays visible rather than being evicted on read.
|
||||
if entry != nil && entry.TtlSec > 0 && !entry.IsDirectory() && !entryLockedForDelete(entry, false, time.Now()) {
|
||||
if entry.IsExpireS3Enabled() {
|
||||
if entry.GetS3ExpireTime().Before(time.Now()) && !entry.IsS3Versioning() {
|
||||
if delErr := f.doDeleteEntryMetaAndData(ctx, entry, true, false, nil); delErr != nil {
|
||||
@@ -812,7 +814,7 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
|
||||
glog.V(1).InfofCtx(ctx, "listing %q canceled: %v", p, ctx.Err())
|
||||
return false, fmt.Errorf("context canceled: %w", ctx.Err())
|
||||
default:
|
||||
if entry.TtlSec > 0 && !entry.IsDirectory() {
|
||||
if entry.TtlSec > 0 && !entry.IsDirectory() && !entryLockedForDelete(entry, false, time.Now()) {
|
||||
if entry.IsExpireS3Enabled() {
|
||||
if entry.GetS3ExpireTime().Before(time.Now()) && !entry.IsS3Versioning() {
|
||||
// Collect for deletion after iteration completes to avoid DB deadlock
|
||||
|
||||
@@ -5,18 +5,21 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
|
||||
MsgFailDelLockedEntry = "fail to delete locked entry"
|
||||
)
|
||||
|
||||
// ErrNonEmptyFolder is a non-recursive delete refused because the folder still
|
||||
@@ -25,6 +28,11 @@ const (
|
||||
// chose, cannot forge one.
|
||||
var ErrNonEmptyFolder = errors.New(MsgFailDelNonEmptyFolder)
|
||||
|
||||
// ErrEntryLocked is a delete refused because the entry carries an active legal
|
||||
// hold or an unexpired retention. It follows ErrNonEmptyFolder's leading-marker
|
||||
// rule for the same reason.
|
||||
var ErrEntryLocked = errors.New(MsgFailDelLockedEntry)
|
||||
|
||||
// DeleteEntryError turns the text of DeleteEntryResponse.Error back into an
|
||||
// error carrying the condition the filer reported. Call it on the response
|
||||
// field, before formatting a path around it.
|
||||
@@ -32,6 +40,9 @@ func DeleteEntryError(msg string) error {
|
||||
if strings.HasPrefix(msg, MsgFailDelNonEmptyFolder) {
|
||||
return &deleteEntryError{msg: msg, cause: ErrNonEmptyFolder}
|
||||
}
|
||||
if strings.HasPrefix(msg, MsgFailDelLockedEntry) {
|
||||
return &deleteEntryError{msg: msg, cause: ErrEntryLocked}
|
||||
}
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
@@ -45,6 +56,14 @@ func IsNonEmptyFolderError(err error) bool {
|
||||
return errors.Is(err, ErrNonEmptyFolder) || strings.HasPrefix(err.Error(), MsgFailDelNonEmptyFolder)
|
||||
}
|
||||
|
||||
// IsEntryLockedError is the ErrEntryLocked counterpart of IsNonEmptyFolderError.
|
||||
func IsEntryLockedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return errors.Is(err, ErrEntryLocked) || strings.HasPrefix(err.Error(), MsgFailDelLockedEntry)
|
||||
}
|
||||
|
||||
type deleteEntryError struct {
|
||||
msg string
|
||||
cause error
|
||||
@@ -80,6 +99,147 @@ func keepsRemoteObject(ctx context.Context) bool {
|
||||
return ctx.Value(keepRemoteObjectKey{}) != nil
|
||||
}
|
||||
|
||||
type bypassGovernanceRetentionKey struct{}
|
||||
|
||||
// WithBypassGovernanceRetention records that the caller is authorized to drop
|
||||
// governance-mode retention on this delete -- the S3 gateway sets it after
|
||||
// s3:BypassGovernanceRetention. Legal hold and compliance retention refuse
|
||||
// the delete regardless.
|
||||
func WithBypassGovernanceRetention(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, bypassGovernanceRetentionKey{}, true)
|
||||
}
|
||||
|
||||
// BypassesGovernanceRetention reports whether ctx carries the governance-bypass
|
||||
// assertion WithBypassGovernanceRetention plants.
|
||||
func BypassesGovernanceRetention(ctx context.Context) bool {
|
||||
return ctx.Value(bypassGovernanceRetentionKey{}) != nil
|
||||
}
|
||||
|
||||
// entryLockedForDelete mirrors the S3 gateway's EntryHasActiveLock on the
|
||||
// entry's extended attributes, except a governance bypass skips only
|
||||
// governance-mode retention. An unparseable retention date fails closed.
|
||||
func entryLockedForDelete(entry *Entry, bypassGovernance bool, now time.Time) bool {
|
||||
ext := entry.Extended
|
||||
if ext == nil {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(string(ext[s3_constants.ExtLegalHoldKey])), s3_constants.LegalHoldOn) {
|
||||
return true
|
||||
}
|
||||
mode := strings.ToUpper(strings.TrimSpace(string(ext[s3_constants.ExtObjectLockModeKey])))
|
||||
if mode != s3_constants.RetentionModeCompliance && mode != s3_constants.RetentionModeGovernance {
|
||||
return false
|
||||
}
|
||||
if bypassGovernance && mode == s3_constants.RetentionModeGovernance {
|
||||
return false
|
||||
}
|
||||
dateBytes, found := ext[s3_constants.ExtRetentionUntilDateKey]
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
timestamp, err := strconv.ParseInt(strings.TrimSpace(string(dateBytes)), 10, 64)
|
||||
if err != nil {
|
||||
glog.Warningf("unparseable retention date %q on %s, treating as locked: %v", dateBytes, entry.FullPath, err)
|
||||
return true
|
||||
}
|
||||
return time.Unix(timestamp, 0).After(now)
|
||||
}
|
||||
|
||||
func (f *Filer) entryDeleteLockError(ctx context.Context, entry *Entry) error {
|
||||
if f.multipartStagingPath(entry.FullPath) {
|
||||
return nil
|
||||
}
|
||||
if entryLockedForDelete(entry, BypassesGovernanceRetention(ctx), time.Now()) {
|
||||
return fmt.Errorf("%w: %s", ErrEntryLocked, entry.FullPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// multipartStagingPath reports whether p is a bucket's .uploads dir or inside
|
||||
// it. createMultipartUpload stamps the object's requested retention and legal
|
||||
// hold on the upload directory so the completed object inherits them, but the
|
||||
// staging area is bookkeeping, not a locked object: finalize, abort, and
|
||||
// s3.clean.uploads all have to be able to remove it, and the gateway's
|
||||
// bucket-lock scan skips it for the same reason.
|
||||
func (f *Filer) multipartStagingPath(p util.FullPath) bool {
|
||||
rel := strings.TrimPrefix(string(p), f.DirBucketsPath+"/")
|
||||
if rel == string(p) {
|
||||
return false
|
||||
}
|
||||
_, rest, found := strings.Cut(rel, "/")
|
||||
return found && (rest == s3_constants.MultipartUploadsFolder || strings.HasPrefix(rest, s3_constants.MultipartUploadsFolder+"/"))
|
||||
}
|
||||
|
||||
// lockedDescendantError walks the subtree once, refusing the delete on the
|
||||
// first entry under an active legal hold or retention. It runs before the
|
||||
// recursive delete emits any side effect, so a refusal cannot arrive after
|
||||
// earlier descendants -- or the ancestor's remote tombstone -- are already
|
||||
// gone, and it covers the store-drop path whose DeleteFolderChildren never
|
||||
// lists the children it removes. The per-page check in the delete walk still
|
||||
// catches a lock planted between this pass and the delete.
|
||||
func (f *Filer) lockedDescendantError(ctx context.Context, entry *Entry) error {
|
||||
lastFileName := ""
|
||||
includeLastFile := false
|
||||
for {
|
||||
entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
|
||||
}
|
||||
for _, sub := range entries {
|
||||
lastFileName = sub.Name()
|
||||
if lockErr := f.entryDeleteLockError(ctx, sub); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
if sub.IsDirectory() {
|
||||
if lockErr := f.lockedDescendantError(ctx, sub); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(entries) < PaginationSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// objectLockEnabledOnEntry mirrors the S3 gateway's IsObjectLockEnabled.
|
||||
func objectLockEnabledOnEntry(entry *Entry) bool {
|
||||
enabled, found := entry.Extended[s3_constants.ExtObjectLockEnabledKey]
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
v := string(enabled)
|
||||
return v == s3_constants.ObjectLockEnabled || v == "true"
|
||||
}
|
||||
|
||||
// mayDescendantsHoldLocks reports whether the subtree under p can hold an
|
||||
// entry under retention or legal hold. The S3 gateway refuses to set either
|
||||
// outside a lock-enabled bucket, so a tree elsewhere can only carry planted
|
||||
// attributes, which the walk still refuses on sight. The pre-pass that makes
|
||||
// refusal atomic is worth its second scan only when locks genuinely apply.
|
||||
func (f *Filer) mayDescendantsHoldLocks(ctx context.Context, p util.FullPath, entry *Entry) bool {
|
||||
if objectLockEnabledOnEntry(entry) {
|
||||
return true
|
||||
}
|
||||
rel := strings.TrimPrefix(string(p), f.DirBucketsPath+"/")
|
||||
if rel == string(p) {
|
||||
// The subtree can also hold locks by containing the buckets dir
|
||||
// itself: deleting it or an ancestor drops each bucket's children
|
||||
// unseen mid-walk.
|
||||
dir := strings.TrimSuffix(string(p), "/")
|
||||
return dir == f.DirBucketsPath || strings.HasPrefix(f.DirBucketsPath, dir+"/")
|
||||
}
|
||||
bucket, _, _ := strings.Cut(rel, "/")
|
||||
if bucket == "" {
|
||||
return false
|
||||
}
|
||||
bucketEntry, err := f.FindEntry(ctx, util.FullPath(f.DirBucketsPath+"/"+bucket))
|
||||
if err != nil {
|
||||
return !errors.Is(err, filer_pb.ErrNotFound)
|
||||
}
|
||||
return bucketEntry != nil && objectLockEnabledOnEntry(bucketEntry)
|
||||
}
|
||||
|
||||
// emptyFolderDeleteContext makes deleting a folder that is empty in the filer
|
||||
// metadata-only when it is under a remote mount. A mount is populated lazily,
|
||||
// so the folder can still hold remote objects never read, and removing its
|
||||
@@ -131,6 +291,15 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter {
|
||||
return nil
|
||||
}
|
||||
// A cluster replaying a delete the origin already authorized must not
|
||||
// second-guess it: the entry's lock state here may legitimately still be
|
||||
// in force (e.g. an expired-elsewhere retention, or a governance bypass
|
||||
// the peer observed before shipping the event).
|
||||
if !isFromOtherCluster {
|
||||
if lockErr := f.entryDeleteLockError(ctx, entry); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
if !isRecursive && entry.IsDirectory() {
|
||||
ctx = f.emptyFolderDeleteContext(ctx, p)
|
||||
}
|
||||
@@ -143,6 +312,16 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
// per entry rather than dropped wholesale with it.
|
||||
dropsCollection := isDeleteCollection && collectionName != ""
|
||||
if entry.IsDirectory() {
|
||||
// The pre-pass is mandatory when the delete ends in a store-level
|
||||
// drop that never lists the children it removes; when they are walked
|
||||
// anyway, the per-page check already refuses a lock on sight, so the
|
||||
// second scan only buys atomicity where locks genuinely apply.
|
||||
childrenDroppedUnseen := isDeleteCollection && (dropsCollection || !shouldDeleteChunks) && f.Store.CanDropWholeBucket() && !objectLockEnabledOnEntry(entry)
|
||||
if isRecursive && !isFromOtherCluster && (childrenDroppedUnseen || f.mayDescendantsHoldLocks(ctx, p, entry)) {
|
||||
if lockErr := f.lockedDescendantError(ctx, entry); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
// delete the folder children, not including the folder itself
|
||||
err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !dropsCollection, isDeleteCollection && (dropsCollection || !shouldDeleteChunks), isFromOtherCluster, signatures, func(hardLinkIds []HardLinkId) error {
|
||||
// A case not handled:
|
||||
@@ -154,7 +333,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(2).InfofCtx(ctx, "delete directory %s: %v", p, err)
|
||||
if errors.Is(err, ErrNonEmptyFolder) {
|
||||
if errors.Is(err, ErrNonEmptyFolder) || errors.Is(err, ErrEntryLocked) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("delete directory %s: %v", p, err)
|
||||
@@ -164,6 +343,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
// delete the file or folder
|
||||
err = f.doDeleteEntryMetaAndData(ctx, entry, shouldDeleteChunks, isFromOtherCluster, signatures)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrEntryLocked) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("delete file %s: %v", p, err)
|
||||
}
|
||||
|
||||
@@ -208,7 +390,9 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
var chunksToDelete []*filer_pb.FileChunk
|
||||
lastFileName := ""
|
||||
includeLastFile := false
|
||||
listedChildren := !isDeletingBucket || !f.Store.CanDropWholeBucket()
|
||||
// A lock-enabled bucket must be walked even when the store could drop it
|
||||
// whole: entries under active retention or legal hold refuse the delete.
|
||||
listedChildren := !isDeletingBucket || !f.Store.CanDropWholeBucket() || objectLockEnabledOnEntry(entry)
|
||||
if listedChildren {
|
||||
for {
|
||||
entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
|
||||
@@ -225,10 +409,26 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
return fmt.Errorf("%w: %s", ErrNonEmptyFolder, entry.FullPath)
|
||||
}
|
||||
|
||||
// Refuse before any side effect of this page (events, remote
|
||||
// deletes, the folder-children sweep): a locked entry anywhere in
|
||||
// it fails the whole delete rather than slipping through the
|
||||
// store-level drop the walk ends in. Refusal is not a recursive
|
||||
// error -- ignoreRecursiveError must not waive a lock.
|
||||
if isRecursive && !isFromOtherCluster {
|
||||
for _, sub := range entries {
|
||||
if lockErr := f.entryDeleteLockError(ctx, sub); lockErr != nil {
|
||||
f.unnoteRemoteDeletion(entry.FullPath, dirTombstoneTs)
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, sub := range entries {
|
||||
lastFileName = sub.Name()
|
||||
var subTombstoneTs int64
|
||||
if !keepsRemoteObject(ctx) {
|
||||
f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())
|
||||
subTombstoneTs = time.Now().UnixNano()
|
||||
f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), subTombstoneTs)
|
||||
}
|
||||
if sub.IsDirectory() {
|
||||
subIsDeletingBucket := f.IsBucket(sub)
|
||||
@@ -242,7 +442,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil && !ignoreRecursiveError {
|
||||
if err != nil && (!ignoreRecursiveError || errors.Is(err, ErrEntryLocked)) {
|
||||
break
|
||||
}
|
||||
f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, sub), nil, shouldDeleteChunks, isFromOtherCluster, nil)
|
||||
@@ -258,7 +458,12 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil && !ignoreRecursiveError {
|
||||
if err != nil && (!ignoreRecursiveError || errors.Is(err, ErrEntryLocked)) {
|
||||
// the delete is abandoned: the tombstones noted for this
|
||||
// subtree and this level would hide children that still
|
||||
// exist
|
||||
f.unnoteRemoteDeletion(sub.FullPath, subTombstoneTs)
|
||||
f.unnoteRemoteDeletion(entry.FullPath, dirTombstoneTs)
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -274,6 +479,14 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
// a non-recursive delete already proved the folder empty above, so sweeping the
|
||||
// children now can only remove entries that raced in after that listing
|
||||
if isRecursive || !listedChildren {
|
||||
if !listedChildren && !isRecursive && !isFromOtherCluster {
|
||||
// A bucket dropped whole sweeps children nothing has listed; a
|
||||
// recursive caller already walked them in the preflight, but a
|
||||
// non-recursive one has not.
|
||||
if lockErr := f.lockedDescendantError(ctx, entry); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
|
||||
return fmt.Errorf("filer store delete: %w", storeDeletionErr)
|
||||
}
|
||||
@@ -289,6 +502,12 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
|
||||
|
||||
glog.V(3).InfofCtx(ctx, "deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
|
||||
|
||||
if !isFromOtherCluster {
|
||||
if lockErr := f.entryDeleteLockError(ctx, entry); lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
}
|
||||
|
||||
if !isFromOtherCluster && !keepsRemoteObject(ctx) {
|
||||
if _, remoteDeletionErr := f.maybeDeleteFromRemote(ctx, entry); remoteDeletionErr != nil {
|
||||
return remoteDeletionErr
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -117,3 +119,330 @@ func TestDeleteFolderKeepsSharedHardLinkChunks(t *testing.T) {
|
||||
require.NoError(t, err, "surviving link must keep the inode record")
|
||||
require.Equal(t, 0, f.pendingDeletionCount())
|
||||
}
|
||||
|
||||
func retentionEntry(mode string, until time.Time) map[string][]byte {
|
||||
return map[string][]byte{
|
||||
s3_constants.ExtObjectLockModeKey: []byte(mode),
|
||||
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(until.Unix(), 10)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestEntryLockedForDelete(t *testing.T) {
|
||||
now := time.Now()
|
||||
future := now.Add(time.Hour)
|
||||
past := now.Add(-time.Hour)
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
ext map[string][]byte
|
||||
bypass bool
|
||||
locked bool
|
||||
}{
|
||||
{"no attributes", nil, false, false},
|
||||
{"legal hold on", map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")}, false, true},
|
||||
{"legal hold bypass still locks", map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("on")}, true, true},
|
||||
{"legal hold off", map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("OFF")}, false, false},
|
||||
{"compliance active", retentionEntry(s3_constants.RetentionModeCompliance, future), false, true},
|
||||
{"compliance bypass still locks", retentionEntry(s3_constants.RetentionModeCompliance, future), true, true},
|
||||
{"compliance expired", retentionEntry(s3_constants.RetentionModeCompliance, past), false, false},
|
||||
{"governance active", retentionEntry(s3_constants.RetentionModeGovernance, future), false, true},
|
||||
{"governance bypass", retentionEntry(s3_constants.RetentionModeGovernance, future), true, false},
|
||||
{"governance expired", retentionEntry(s3_constants.RetentionModeGovernance, past), false, false},
|
||||
{"mode without date", map[string][]byte{s3_constants.ExtObjectLockModeKey: []byte(s3_constants.RetentionModeCompliance)}, false, false},
|
||||
{"unparseable date fails closed", map[string][]byte{
|
||||
s3_constants.ExtObjectLockModeKey: []byte(s3_constants.RetentionModeCompliance),
|
||||
s3_constants.ExtRetentionUntilDateKey: []byte("not-a-time"),
|
||||
}, false, true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
entry := &Entry{FullPath: "/buckets/b/o", Attr: Attr{Mode: 0644}, Extended: tt.ext}
|
||||
require.Equal(t, tt.locked, entryLockedForDelete(entry, tt.bypass, now))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The same leading-marker contract as ErrNonEmptyFolder keeps the wire text
|
||||
// classifiable and unforgeable.
|
||||
func TestEntryLockedClassification(t *testing.T) {
|
||||
err := fmt.Errorf("%w: %s", ErrEntryLocked, "/buckets/b/locked.bin")
|
||||
if !IsEntryLockedError(err) {
|
||||
t.Errorf("expected the sentinel to be recognized: %v", err)
|
||||
}
|
||||
if !errors.Is(DeleteEntryError(err.Error()), ErrEntryLocked) {
|
||||
t.Errorf("expected the wire text to classify: %v", err)
|
||||
}
|
||||
spoofed := "delete file /buckets/b/" + MsgFailDelLockedEntry + ": filer store delete: disk full"
|
||||
if errors.Is(DeleteEntryError(spoofed), ErrEntryLocked) {
|
||||
t.Errorf("expected no forgery from the entry name: %v", spoofed)
|
||||
}
|
||||
if IsEntryLockedError(errors.New(spoofed)) {
|
||||
t.Errorf("expected no forgery from the entry name: %v", spoofed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteEntryRefusesLockedObject(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
|
||||
locked := &Entry{
|
||||
FullPath: "/buckets/b/locked.bin",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, time.Now().Add(time.Hour)),
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, locked, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, locked.FullPath, false, false, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.True(t, IsEntryLockedError(err))
|
||||
require.True(t, IsEntryLockedError(DeleteEntryError(err.Error())))
|
||||
require.NotNil(t, store.getEntry("/buckets/b/locked.bin"), "locked entry must survive the refused delete")
|
||||
}
|
||||
|
||||
// A recursive delete fails at the locked descendant without removing anything.
|
||||
func TestDeleteEntryRecursiveRefusesLockedDescendant(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/d", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/d/free.txt", Attr: Attr{Mode: 0644}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/d/locked.txt",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/d", true, true, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/d"), "the folder itself must survive")
|
||||
require.NotNil(t, store.getEntry("/d/free.txt"), "unlocked siblings must survive a refused delete")
|
||||
require.NotNil(t, store.getEntry("/d/locked.txt"))
|
||||
}
|
||||
|
||||
// A lock refusal surfaces from any depth and cannot be waived by
|
||||
// ignoreRecursiveError: nothing in the subtree may be removed.
|
||||
func TestDeleteEntryRecursiveRefusesLockedGrandchild(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/d", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/d/early.txt", Attr: Attr{Mode: 0644}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/d/sub", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/d/sub/locked.txt",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, time.Now().Add(time.Hour)),
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/d", true, true, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
for _, p := range []string{"/d", "/d/early.txt", "/d/sub", "/d/sub/locked.txt"} {
|
||||
require.NotNil(t, store.getEntry(p), "%s must survive a refused recursive delete", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Governance bypass is the only waiver the delete honors.
|
||||
func TestDeleteEntryGovernanceBypass(t *testing.T) {
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
t.Run("governance locked deletes with bypass", func(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
entry := &Entry{FullPath: "/o", Attr: Attr{Mode: 0644}, Extended: retentionEntry(s3_constants.RetentionModeGovernance, future)}
|
||||
require.NoError(t, f.CreateEntry(context.Background(), entry, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(WithBypassGovernanceRetention(context.Background()), entry.FullPath, false, false, true, false, nil, 0))
|
||||
require.Nil(t, store.getEntry("/o"))
|
||||
})
|
||||
|
||||
t.Run("compliance still refuses with bypass", func(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
entry := &Entry{FullPath: "/o", Attr: Attr{Mode: 0644}, Extended: retentionEntry(s3_constants.RetentionModeCompliance, future)}
|
||||
require.NoError(t, f.CreateEntry(context.Background(), entry, nil, false, false, nil, true, 255))
|
||||
require.ErrorIs(t, f.DeleteEntryMetaAndData(WithBypassGovernanceRetention(context.Background()), entry.FullPath, false, false, true, false, nil, 0), ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/o"))
|
||||
})
|
||||
}
|
||||
|
||||
// A delete replayed from a peer cluster applies as sent: the origin already
|
||||
// enforced retention before shipping the event.
|
||||
func TestDeleteEntryClusterReplaySkipsLockCheck(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: "/locked",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, entry, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(ctx, entry.FullPath, false, false, true, true, nil, 0))
|
||||
require.Nil(t, store.getEntry("/locked"))
|
||||
}
|
||||
|
||||
type bucketDropStore struct {
|
||||
*stubFilerStore
|
||||
}
|
||||
|
||||
func (s *bucketDropStore) OnBucketCreation(string) {}
|
||||
func (s *bucketDropStore) OnBucketDeletion(string) {}
|
||||
func (s *bucketDropStore) CanDropWholeBucket() bool { return true }
|
||||
|
||||
// A lock-enabled bucket is walked rather than dropped whole, so its locked
|
||||
// objects still refuse the delete on a store that could drop it blindly.
|
||||
func TestDeleteBucketOnDropCapableStoreRefusesLockedObject(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
f.Store = NewFilerStoreWrapper(&bucketDropStore{store})
|
||||
ctx := context.Background()
|
||||
|
||||
bucket := &Entry{
|
||||
FullPath: "/buckets/b",
|
||||
Attr: Attr{Mode: os.ModeDir | 0755},
|
||||
Extended: map[string][]byte{s3_constants.ExtObjectLockEnabledKey: []byte(s3_constants.ObjectLockEnabled)},
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, bucket, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/buckets/b/locked.bin",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, time.Now().Add(time.Hour)),
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/buckets/b", true, false, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/buckets/b/locked.bin"), "a drop-whole-bucket store must not swallow the locked object")
|
||||
}
|
||||
|
||||
// A bucket without the lock-enabled attribute can still carry entries under
|
||||
// retention, so the drop-whole-bucket path must not skip the lock walk.
|
||||
func TestDeleteUnmarkedBucketOnDropCapableStoreRefusesLockedObject(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
f.Store = NewFilerStoreWrapper(&bucketDropStore{store})
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/buckets/b", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/buckets/b/held.bin",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/buckets/b", true, true, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/buckets/b/held.bin"), "an unmarked bucket drop must not swallow the locked object")
|
||||
}
|
||||
|
||||
// A non-recursive delete of an unmarked bucket never lists the children the
|
||||
// store drops, so the sweep itself must check them for locks first.
|
||||
func TestDeleteUnmarkedBucketNonRecursiveRefusesLockedObject(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
f.Store = NewFilerStoreWrapper(&bucketDropStore{store})
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/buckets/b", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/buckets/b/held.bin",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/buckets/b", false, true, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/buckets/b/held.bin"), "a non-recursive unmarked bucket drop must not swallow the locked object")
|
||||
}
|
||||
|
||||
// A recursive delete rooted at the buckets dir reaches each bucket only
|
||||
// mid-walk, so the pre-pass has to know the subtree holds buckets whose
|
||||
// children the store would drop unseen.
|
||||
func TestDeleteBucketsRootRecursiveRefusesLockedObject(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
f.Store = NewFilerStoreWrapper(&bucketDropStore{store})
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/buckets", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: "/buckets/b", Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{
|
||||
FullPath: "/buckets/b/held.bin",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, "/buckets", true, true, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry("/buckets/b/held.bin"), "an unseen bucket drop must not swallow the locked object")
|
||||
}
|
||||
|
||||
// An expired retention no longer locks: the delete proceeds normally.
|
||||
func TestDeleteEntryAllowsExpiredRetention(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: "/old",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, time.Now().Add(-time.Hour)),
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, entry, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(ctx, entry.FullPath, false, false, true, false, nil, 0))
|
||||
require.Nil(t, store.getEntry("/old"))
|
||||
}
|
||||
|
||||
// Retention and legal hold stamped on a multipart upload directory are
|
||||
// bookkeeping for the finished object, not a lock on the staging entry:
|
||||
// finalize, abort, and s3.clean.uploads all remove it. The gateway's
|
||||
// bucket-lock scan skips .uploads for the same reason.
|
||||
func TestDeleteEntryAllowsLockedMultipartStaging(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
ctx := context.Background()
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
upload := &Entry{
|
||||
FullPath: "/buckets/b/.uploads/upload-id",
|
||||
Attr: Attr{Mode: os.ModeDir | 0755},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, future),
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, upload, nil, false, false, nil, true, 255))
|
||||
part := &Entry{
|
||||
FullPath: "/buckets/b/.uploads/upload-id/0001_part",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("ON")},
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, part, nil, false, false, nil, true, 255))
|
||||
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(ctx, upload.FullPath, true, false, true, false, nil, 0))
|
||||
require.Nil(t, store.getEntry(string(upload.FullPath)))
|
||||
require.Nil(t, store.getEntry(string(part.FullPath)))
|
||||
}
|
||||
|
||||
// The .uploads carve-out stops at the bucket's first path segment: an object
|
||||
// key that nests the name deeper still holds real lock attributes.
|
||||
func TestDeleteEntryRefusesLockedObjectUnderNestedUploadsName(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
f.DirBucketsPath = "/buckets"
|
||||
ctx := context.Background()
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: "/buckets/b/dir/.uploads/x",
|
||||
Attr: Attr{Mode: 0644},
|
||||
Extended: retentionEntry(s3_constants.RetentionModeCompliance, time.Now().Add(time.Hour)),
|
||||
}
|
||||
require.NoError(t, f.CreateEntry(ctx, entry, nil, false, false, nil, true, 255))
|
||||
|
||||
err := f.DeleteEntryMetaAndData(ctx, entry.FullPath, false, false, true, false, nil, 0)
|
||||
require.ErrorIs(t, err, ErrEntryLocked)
|
||||
require.NotNil(t, store.getEntry(string(entry.FullPath)))
|
||||
}
|
||||
@@ -326,6 +326,7 @@ enum FilerError {
|
||||
ENTRY_ALREADY_EXISTS = 5; // O_EXCL and entry already exists
|
||||
PRECONDITION_FAILED = 6; // WriteCondition not satisfied
|
||||
READ_ONLY = 7; // path matches a read-only storage rule (e.g. bucket over quota)
|
||||
ENTRY_LOCKED = 8; // mutation refused by an active legal hold or retention
|
||||
}
|
||||
|
||||
// ObjectMutation is one entry-level change applied by ObjectTransaction. All
|
||||
@@ -355,6 +356,7 @@ message ObjectMutation {
|
||||
bool touch_mtime = 12; // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
bool remove_empty_parent = 13; // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
bool keep_remote_object = 14; // DELETE: leave the object on the mounted remote storage
|
||||
bool bypass_governance_retention = 15; // DELETE: skip governance-mode retention guard (legal hold / compliance still refuse)
|
||||
}
|
||||
|
||||
// Recompute re-derives a pointer entry (directory/name on the mutation) from the
|
||||
@@ -522,6 +524,10 @@ message DeleteEntryRequest {
|
||||
repeated int32 signatures = 8;
|
||||
int64 if_not_modified_after = 9;
|
||||
bool keep_remote_object = 10;
|
||||
// Caller asserts governance-retention bypass authority (e.g. the S3 gateway
|
||||
// after s3:BypassGovernanceRetention). Only governance-mode retention yields;
|
||||
// legal hold and compliance retention still refuse the delete.
|
||||
bool bypass_governance_retention = 11;
|
||||
}
|
||||
|
||||
message DeleteEntryResponse {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.36.6
|
||||
// protoc v7.35.0
|
||||
// protoc v3.21.12
|
||||
// source: filer.proto
|
||||
|
||||
package filer_pb
|
||||
@@ -86,6 +86,7 @@ const (
|
||||
FilerError_ENTRY_ALREADY_EXISTS FilerError = 5 // O_EXCL and entry already exists
|
||||
FilerError_PRECONDITION_FAILED FilerError = 6 // WriteCondition not satisfied
|
||||
FilerError_READ_ONLY FilerError = 7 // path matches a read-only storage rule (e.g. bucket over quota)
|
||||
FilerError_ENTRY_LOCKED FilerError = 8 // mutation refused by an active legal hold or retention
|
||||
)
|
||||
|
||||
// Enum value maps for FilerError.
|
||||
@@ -99,6 +100,7 @@ var (
|
||||
5: "ENTRY_ALREADY_EXISTS",
|
||||
6: "PRECONDITION_FAILED",
|
||||
7: "READ_ONLY",
|
||||
8: "ENTRY_LOCKED",
|
||||
}
|
||||
FilerError_value = map[string]int32{
|
||||
"OK": 0,
|
||||
@@ -109,6 +111,7 @@ var (
|
||||
"ENTRY_ALREADY_EXISTS": 5,
|
||||
"PRECONDITION_FAILED": 6,
|
||||
"READ_ONLY": 7,
|
||||
"ENTRY_LOCKED": 8,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1535,23 +1538,24 @@ func (x *WriteCondition) GetClauses() []*WriteCondition_Clause {
|
||||
// several RPCs. Data-bearing writes (entries with chunks) should be written
|
||||
// before the transaction; mutations here are metadata-scoped.
|
||||
type ObjectMutation struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Type ObjectMutation_Type `protobuf:"varint,1,opt,name=type,proto3,enum=filer_pb.ObjectMutation_Type" json:"type,omitempty"`
|
||||
Directory string `protobuf:"bytes,2,opt,name=directory,proto3" json:"directory,omitempty"`
|
||||
Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` // entry name for DELETE / PATCH_EXTENDED / RECOMPUTE_LATEST (the pointer entry)
|
||||
Entry *Entry `protobuf:"bytes,4,opt,name=entry,proto3" json:"entry,omitempty"` // full entry for PUT
|
||||
SetExtended map[string][]byte `protobuf:"bytes,5,rep,name=set_extended,json=setExtended,proto3" json:"set_extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // PATCH_EXTENDED: keys to set
|
||||
DeleteExtended []string `protobuf:"bytes,6,rep,name=delete_extended,json=deleteExtended,proto3" json:"delete_extended,omitempty"` // PATCH_EXTENDED: keys to remove
|
||||
IsDeleteData bool `protobuf:"varint,7,opt,name=is_delete_data,json=isDeleteData,proto3" json:"is_delete_data,omitempty"` // DELETE: also delete chunk data
|
||||
IsRecursive bool `protobuf:"varint,8,opt,name=is_recursive,json=isRecursive,proto3" json:"is_recursive,omitempty"` // DELETE: recurse into a directory
|
||||
Recompute *Recompute `protobuf:"bytes,9,opt,name=recompute,proto3" json:"recompute,omitempty"` // RECOMPUTE_LATEST parameters
|
||||
SetContent bool `protobuf:"varint,10,opt,name=set_content,json=setContent,proto3" json:"set_content,omitempty"` // PATCH_EXTENDED: replace Entry.content with content
|
||||
Content []byte `protobuf:"bytes,11,opt,name=content,proto3" json:"content,omitempty"` // PATCH_EXTENDED: new Entry.content when set_content
|
||||
TouchMtime bool `protobuf:"varint,12,opt,name=touch_mtime,json=touchMtime,proto3" json:"touch_mtime,omitempty"` // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
RemoveEmptyParent bool `protobuf:"varint,13,opt,name=remove_empty_parent,json=removeEmptyParent,proto3" json:"remove_empty_parent,omitempty"` // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
KeepRemoteObject bool `protobuf:"varint,14,opt,name=keep_remote_object,json=keepRemoteObject,proto3" json:"keep_remote_object,omitempty"` // DELETE: leave the object on the mounted remote storage
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Type ObjectMutation_Type `protobuf:"varint,1,opt,name=type,proto3,enum=filer_pb.ObjectMutation_Type" json:"type,omitempty"`
|
||||
Directory string `protobuf:"bytes,2,opt,name=directory,proto3" json:"directory,omitempty"`
|
||||
Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` // entry name for DELETE / PATCH_EXTENDED / RECOMPUTE_LATEST (the pointer entry)
|
||||
Entry *Entry `protobuf:"bytes,4,opt,name=entry,proto3" json:"entry,omitempty"` // full entry for PUT
|
||||
SetExtended map[string][]byte `protobuf:"bytes,5,rep,name=set_extended,json=setExtended,proto3" json:"set_extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // PATCH_EXTENDED: keys to set
|
||||
DeleteExtended []string `protobuf:"bytes,6,rep,name=delete_extended,json=deleteExtended,proto3" json:"delete_extended,omitempty"` // PATCH_EXTENDED: keys to remove
|
||||
IsDeleteData bool `protobuf:"varint,7,opt,name=is_delete_data,json=isDeleteData,proto3" json:"is_delete_data,omitempty"` // DELETE: also delete chunk data
|
||||
IsRecursive bool `protobuf:"varint,8,opt,name=is_recursive,json=isRecursive,proto3" json:"is_recursive,omitempty"` // DELETE: recurse into a directory
|
||||
Recompute *Recompute `protobuf:"bytes,9,opt,name=recompute,proto3" json:"recompute,omitempty"` // RECOMPUTE_LATEST parameters
|
||||
SetContent bool `protobuf:"varint,10,opt,name=set_content,json=setContent,proto3" json:"set_content,omitempty"` // PATCH_EXTENDED: replace Entry.content with content
|
||||
Content []byte `protobuf:"bytes,11,opt,name=content,proto3" json:"content,omitempty"` // PATCH_EXTENDED: new Entry.content when set_content
|
||||
TouchMtime bool `protobuf:"varint,12,opt,name=touch_mtime,json=touchMtime,proto3" json:"touch_mtime,omitempty"` // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
RemoveEmptyParent bool `protobuf:"varint,13,opt,name=remove_empty_parent,json=removeEmptyParent,proto3" json:"remove_empty_parent,omitempty"` // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
KeepRemoteObject bool `protobuf:"varint,14,opt,name=keep_remote_object,json=keepRemoteObject,proto3" json:"keep_remote_object,omitempty"` // DELETE: leave the object on the mounted remote storage
|
||||
BypassGovernanceRetention bool `protobuf:"varint,15,opt,name=bypass_governance_retention,json=bypassGovernanceRetention,proto3" json:"bypass_governance_retention,omitempty"` // DELETE: skip governance-mode retention guard (legal hold / compliance still refuse)
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ObjectMutation) Reset() {
|
||||
@@ -1682,6 +1686,13 @@ func (x *ObjectMutation) GetKeepRemoteObject() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *ObjectMutation) GetBypassGovernanceRetention() bool {
|
||||
if x != nil {
|
||||
return x.BypassGovernanceRetention
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Recompute re-derives a pointer entry (directory/name on the mutation) from the
|
||||
// current contents of a scanned directory, atomically under the transaction's
|
||||
// lock. It is mechanical: the filer picks the child that sorts first or last by
|
||||
@@ -2771,8 +2782,12 @@ type DeleteEntryRequest struct {
|
||||
Signatures []int32 `protobuf:"varint,8,rep,packed,name=signatures,proto3" json:"signatures,omitempty"`
|
||||
IfNotModifiedAfter int64 `protobuf:"varint,9,opt,name=if_not_modified_after,json=ifNotModifiedAfter,proto3" json:"if_not_modified_after,omitempty"`
|
||||
KeepRemoteObject bool `protobuf:"varint,10,opt,name=keep_remote_object,json=keepRemoteObject,proto3" json:"keep_remote_object,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// Caller asserts governance-retention bypass authority (e.g. the S3 gateway
|
||||
// after s3:BypassGovernanceRetention). Only governance-mode retention yields;
|
||||
// legal hold and compliance retention still refuse the delete.
|
||||
BypassGovernanceRetention bool `protobuf:"varint,11,opt,name=bypass_governance_retention,json=bypassGovernanceRetention,proto3" json:"bypass_governance_retention,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *DeleteEntryRequest) Reset() {
|
||||
@@ -2868,6 +2883,13 @@ func (x *DeleteEntryRequest) GetKeepRemoteObject() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *DeleteEntryRequest) GetBypassGovernanceRetention() bool {
|
||||
if x != nil {
|
||||
return x.BypassGovernanceRetention
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type DeleteEntryResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
|
||||
@@ -7416,7 +7438,7 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\x18IF_EXTENDED_TIME_ELAPSED\x10\b\x12\x13\n" +
|
||||
"\x0fIF_CHUNKS_EQUAL\x10\t\x12\x12\n" +
|
||||
"\x0eIF_ENTRY_EQUAL\x10\n" +
|
||||
"\"\xd0\x05\n" +
|
||||
"\"\x90\x06\n" +
|
||||
"\x0eObjectMutation\x121\n" +
|
||||
"\x04type\x18\x01 \x01(\x0e2\x1d.filer_pb.ObjectMutation.TypeR\x04type\x12\x1c\n" +
|
||||
"\tdirectory\x18\x02 \x01(\tR\tdirectory\x12\x12\n" +
|
||||
@@ -7434,7 +7456,8 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\vtouch_mtime\x18\f \x01(\bR\n" +
|
||||
"touchMtime\x12.\n" +
|
||||
"\x13remove_empty_parent\x18\r \x01(\bR\x11removeEmptyParent\x12,\n" +
|
||||
"\x12keep_remote_object\x18\x0e \x01(\bR\x10keepRemoteObject\x1a>\n" +
|
||||
"\x12keep_remote_object\x18\x0e \x01(\bR\x10keepRemoteObject\x12>\n" +
|
||||
"\x1bbypass_governance_retention\x18\x0f \x01(\bR\x19bypassGovernanceRetention\x1a>\n" +
|
||||
"\x10SetExtendedEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"E\n" +
|
||||
@@ -7539,7 +7562,7 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"entry_name\x18\x02 \x01(\tR\tentryName\x12+\n" +
|
||||
"\x06chunks\x18\x03 \x03(\v2\x13.filer_pb.FileChunkR\x06chunks\"\x17\n" +
|
||||
"\x15AppendToEntryResponse\"\xf9\x02\n" +
|
||||
"\x15AppendToEntryResponse\"\xb9\x03\n" +
|
||||
"\x12DeleteEntryRequest\x12\x1c\n" +
|
||||
"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" +
|
||||
"\x04name\x18\x02 \x01(\tR\x04name\x12$\n" +
|
||||
@@ -7552,7 +7575,8 @@ const file_filer_proto_rawDesc = "" +
|
||||
"signatures\x121\n" +
|
||||
"\x15if_not_modified_after\x18\t \x01(\x03R\x12ifNotModifiedAfter\x12,\n" +
|
||||
"\x12keep_remote_object\x18\n" +
|
||||
" \x01(\bR\x10keepRemoteObject\"w\n" +
|
||||
" \x01(\bR\x10keepRemoteObject\x12>\n" +
|
||||
"\x1bbypass_governance_retention\x18\v \x01(\bR\x19bypassGovernanceRetention\"w\n" +
|
||||
"\x13DeleteEntryResponse\x12\x14\n" +
|
||||
"\x05error\x18\x01 \x01(\tR\x05error\x12J\n" +
|
||||
"\x0emetadata_event\x18\x02 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\"\xed\x01\n" +
|
||||
@@ -7918,7 +7942,7 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\x05SSE_C\x10\x01\x12\v\n" +
|
||||
"\aSSE_KMS\x10\x02\x12\n" +
|
||||
"\n" +
|
||||
"\x06SSE_S3\x10\x03*\xb4\x01\n" +
|
||||
"\x06SSE_S3\x10\x03*\xc6\x01\n" +
|
||||
"\n" +
|
||||
"FilerError\x12\x06\n" +
|
||||
"\x02OK\x10\x00\x12\x17\n" +
|
||||
@@ -7928,7 +7952,8 @@ const file_filer_proto_rawDesc = "" +
|
||||
"\x10EXISTING_IS_FILE\x10\x04\x12\x18\n" +
|
||||
"\x14ENTRY_ALREADY_EXISTS\x10\x05\x12\x17\n" +
|
||||
"\x13PRECONDITION_FAILED\x10\x06\x12\r\n" +
|
||||
"\tREAD_ONLY\x10\a*u\n" +
|
||||
"\tREAD_ONLY\x10\a\x12\x10\n" +
|
||||
"\fENTRY_LOCKED\x10\b*u\n" +
|
||||
"\vPosixLockOp\x12\f\n" +
|
||||
"\bTRY_LOCK\x10\x00\x12\n" +
|
||||
"\n" +
|
||||
|
||||
@@ -1289,6 +1289,16 @@ func (m *ObjectMutation) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
|
||||
i -= len(m.unknownFields)
|
||||
copy(dAtA[i:], m.unknownFields)
|
||||
}
|
||||
if m.BypassGovernanceRetention {
|
||||
i--
|
||||
if m.BypassGovernanceRetention {
|
||||
dAtA[i] = 1
|
||||
} else {
|
||||
dAtA[i] = 0
|
||||
}
|
||||
i--
|
||||
dAtA[i] = 0x78
|
||||
}
|
||||
if m.KeepRemoteObject {
|
||||
i--
|
||||
if m.KeepRemoteObject {
|
||||
@@ -2486,6 +2496,16 @@ func (m *DeleteEntryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
|
||||
i -= len(m.unknownFields)
|
||||
copy(dAtA[i:], m.unknownFields)
|
||||
}
|
||||
if m.BypassGovernanceRetention {
|
||||
i--
|
||||
if m.BypassGovernanceRetention {
|
||||
dAtA[i] = 1
|
||||
} else {
|
||||
dAtA[i] = 0
|
||||
}
|
||||
i--
|
||||
dAtA[i] = 0x58
|
||||
}
|
||||
if m.KeepRemoteObject {
|
||||
i--
|
||||
if m.KeepRemoteObject {
|
||||
@@ -2659,6 +2679,16 @@ func (m *AtomicRenameEntryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, err
|
||||
i -= len(m.unknownFields)
|
||||
copy(dAtA[i:], m.unknownFields)
|
||||
}
|
||||
if m.FailIfTargetExists {
|
||||
i--
|
||||
if m.FailIfTargetExists {
|
||||
dAtA[i] = 1
|
||||
} else {
|
||||
dAtA[i] = 0
|
||||
}
|
||||
i--
|
||||
dAtA[i] = 0x30
|
||||
}
|
||||
if len(m.Signatures) > 0 {
|
||||
var pksize2 int
|
||||
for _, num := range m.Signatures {
|
||||
@@ -7108,6 +7138,9 @@ func (m *ObjectMutation) SizeVT() (n int) {
|
||||
if m.KeepRemoteObject {
|
||||
n += 2
|
||||
}
|
||||
if m.BypassGovernanceRetention {
|
||||
n += 2
|
||||
}
|
||||
n += len(m.unknownFields)
|
||||
return n
|
||||
}
|
||||
@@ -7548,6 +7581,9 @@ func (m *DeleteEntryRequest) SizeVT() (n int) {
|
||||
if m.KeepRemoteObject {
|
||||
n += 2
|
||||
}
|
||||
if m.BypassGovernanceRetention {
|
||||
n += 2
|
||||
}
|
||||
n += len(m.unknownFields)
|
||||
return n
|
||||
}
|
||||
@@ -7599,6 +7635,9 @@ func (m *AtomicRenameEntryRequest) SizeVT() (n int) {
|
||||
}
|
||||
n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l
|
||||
}
|
||||
if m.FailIfTargetExists {
|
||||
n += 2
|
||||
}
|
||||
n += len(m.unknownFields)
|
||||
return n
|
||||
}
|
||||
@@ -13106,6 +13145,26 @@ func (m *ObjectMutation) UnmarshalVT(dAtA []byte) error {
|
||||
}
|
||||
}
|
||||
m.KeepRemoteObject = bool(v != 0)
|
||||
case 15:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field BypassGovernanceRetention", wireType)
|
||||
}
|
||||
var v int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return protohelpers.ErrIntOverflow
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
v |= int(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
m.BypassGovernanceRetention = bool(v != 0)
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := protohelpers.Skip(dAtA[iNdEx:])
|
||||
@@ -16157,6 +16216,26 @@ func (m *DeleteEntryRequest) UnmarshalVT(dAtA []byte) error {
|
||||
}
|
||||
}
|
||||
m.KeepRemoteObject = bool(v != 0)
|
||||
case 11:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field BypassGovernanceRetention", wireType)
|
||||
}
|
||||
var v int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return protohelpers.ErrIntOverflow
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
v |= int(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
m.BypassGovernanceRetention = bool(v != 0)
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := protohelpers.Skip(dAtA[iNdEx:])
|
||||
@@ -16531,6 +16610,26 @@ func (m *AtomicRenameEntryRequest) UnmarshalVT(dAtA []byte) error {
|
||||
} else {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Signatures", wireType)
|
||||
}
|
||||
case 6:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field FailIfTargetExists", wireType)
|
||||
}
|
||||
var v int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return protohelpers.ErrIntOverflow
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
v |= int(b&0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
m.FailIfTargetExists = bool(v != 0)
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := protohelpers.Skip(dAtA[iNdEx:])
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -163,12 +164,18 @@ func deleteObjectEntry(ctx context.Context, client filer_pb.SeaweedFilerClient,
|
||||
}
|
||||
|
||||
func doDeleteEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
|
||||
bypass := filer.BypassesGovernanceRetention(ctx)
|
||||
if bypass {
|
||||
// the filer honors the flag only for an authenticated admin peer
|
||||
ctx, _ = credential.WithS3InternalAdminAuth(ctx)
|
||||
}
|
||||
request := &filer_pb.DeleteEntryRequest{
|
||||
Directory: parentDirectoryPath,
|
||||
Name: entryName,
|
||||
IsDeleteData: isDeleteData,
|
||||
IsRecursive: isRecursive,
|
||||
IgnoreRecursiveError: true,
|
||||
Directory: parentDirectoryPath,
|
||||
Name: entryName,
|
||||
IsDeleteData: isDeleteData,
|
||||
IsRecursive: isRecursive,
|
||||
IgnoreRecursiveError: true,
|
||||
BypassGovernanceRetention: bypass,
|
||||
}
|
||||
|
||||
glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
|
||||
|
||||
@@ -502,6 +502,12 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
|
||||
// it keeps a shared one rather than risk another bucket's volumes.
|
||||
err := s3a.rm(r.Context(), s3a.option.BucketsPath, bucket, false, true)
|
||||
if err != nil {
|
||||
// A lock that landed after the check above reads the same to the
|
||||
// client as one it found: the bucket cannot be removed yet.
|
||||
if filer.IsEntryLockedError(err) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrBucketNotEmpty)
|
||||
return
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -93,20 +94,28 @@ func (s3a *S3ApiServer) deleteDirectoryMarker(r *http.Request, bucket, object st
|
||||
break
|
||||
}
|
||||
}
|
||||
if rmErr := s3a.rm(r.Context(), markerDir, s3_constants.VersionsFolder, true, true); rmErr != nil {
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if rmErr := s3a.rm(ctx, markerDir, s3_constants.VersionsFolder, true, true); rmErr != nil {
|
||||
glog.Errorf("deleteDirectoryMarker: failed to remove stale history of %s/%s: %v", bucket, object, rmErr)
|
||||
return s3err.ErrInternalError
|
||||
return deleteEntryErrCode(rmErr)
|
||||
}
|
||||
case !errors.Is(historyErr, filer_pb.ErrNotFound):
|
||||
glog.Errorf("deleteDirectoryMarker: cannot read stale history of %s/%s: %v", bucket, object, historyErr)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, object, false)
|
||||
return s3a.deleteUnversionedObjectWithClient(ctx, client, bucket, object, false)
|
||||
}); err != nil {
|
||||
glog.Errorf("deleteDirectoryMarker: failed to delete %s/%s: %v", bucket, object, err)
|
||||
return s3err.ErrInternalError
|
||||
return deleteEntryErrCode(err)
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -25,6 +26,16 @@ type deleteMutationResult struct {
|
||||
deleteMarker bool
|
||||
}
|
||||
|
||||
// The filer re-checks object lock during a delete; a refusal means a lock
|
||||
// landed between the gateway's own check and the filer's, so it maps to the
|
||||
// same AccessDenied the check would have returned.
|
||||
func deleteEntryErrCode(err error) s3err.ErrorCode {
|
||||
if filer.IsEntryLockedError(err) {
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
|
||||
func deleteErrorFromCode(code s3err.ErrorCode, key, versionId string) DeleteError {
|
||||
apiErr := s3err.GetAPIError(code)
|
||||
return DeleteError{
|
||||
@@ -143,9 +154,13 @@ func (s3a *S3ApiServer) deleteVersionedObject(r *http.Request, bucket, object, v
|
||||
glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s version %s: %v", bucket, object, versionId, err)
|
||||
return result, s3err.ErrAccessDenied
|
||||
}
|
||||
if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, versionId, false); err != nil {
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if err := s3a.deleteSpecificObjectVersion(ctx, bucket, object, versionId, false); err != nil {
|
||||
glog.Errorf("deleteVersionedObject: failed to delete specific version %s for %s/%s: %v", versionId, bucket, object, err)
|
||||
return result, s3err.ErrInternalError
|
||||
return result, deleteEntryErrCode(err)
|
||||
}
|
||||
result.versionId = versionId
|
||||
return result, s3err.ErrNone
|
||||
@@ -166,9 +181,13 @@ func (s3a *S3ApiServer) deleteVersionedObject(r *http.Request, bucket, object, v
|
||||
glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s null version: %v", bucket, object, err)
|
||||
return result, s3err.ErrAccessDenied
|
||||
}
|
||||
if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, "null", false); err != nil {
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if err := s3a.deleteSpecificObjectVersion(ctx, bucket, object, "null", false); err != nil {
|
||||
glog.Errorf("deleteVersionedObject: failed to delete null version for %s/%s: %v", bucket, object, err)
|
||||
return result, s3err.ErrInternalError
|
||||
return result, deleteEntryErrCode(err)
|
||||
}
|
||||
// Suspended versioning overwrites the null version with a single null delete
|
||||
// marker (S3 spec), so the marker replaces any prior one instead of piling up.
|
||||
@@ -343,11 +362,15 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, object, false)
|
||||
return s3a.deleteUnversionedObjectWithClient(ctx, client, bucket, object, false)
|
||||
}); err != nil {
|
||||
glog.Errorf("DeleteObjectHandler: failed to delete %s/%s: %v", bucket, object, err)
|
||||
return s3err.ErrInternalError
|
||||
return deleteEntryErrCode(err)
|
||||
}
|
||||
|
||||
return s3err.ErrNone
|
||||
@@ -534,9 +557,13 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
return s3a.deleteDirectoryMarker(r, bucket, key)
|
||||
}
|
||||
|
||||
if err := s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, key, false); err != nil {
|
||||
ctx := r.Context()
|
||||
if governanceBypassAllowed {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
}
|
||||
if err := s3a.deleteUnversionedObjectWithClient(ctx, client, bucket, key, false); err != nil {
|
||||
glog.Errorf("DeleteMultipleObjectsHandler: failed to delete %s/%s: %v", bucket, key, err)
|
||||
return s3err.ErrInternalError
|
||||
return deleteEntryErrCode(err)
|
||||
}
|
||||
|
||||
return s3err.ErrNone
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -167,10 +168,18 @@ func (s3a *S3ApiServer) objectTxnOnFiler(owner pb.ServerAddress, req *filer_pb.O
|
||||
if req.RequestId == "" {
|
||||
req.RequestId = uuid.NewString()
|
||||
}
|
||||
txnCtx := context.Background()
|
||||
for _, m := range req.Mutations {
|
||||
if m.BypassGovernanceRetention {
|
||||
// the filer honors the flag only for an authenticated admin peer
|
||||
txnCtx, _ = credential.WithS3InternalAdminAuth(txnCtx)
|
||||
break
|
||||
}
|
||||
}
|
||||
var resp *filer_pb.ObjectTransactionResponse
|
||||
txn := func(client filer_pb.SeaweedFilerClient) error {
|
||||
var e error
|
||||
resp, e = client.ObjectTransaction(context.Background(), req)
|
||||
resp, e = client.ObjectTransaction(txnCtx, req)
|
||||
return e
|
||||
}
|
||||
if s3a.filerClient == nil {
|
||||
|
||||
@@ -194,7 +194,7 @@ func (s3a *S3ApiServer) routedDeleteSpecificVersion(owner pb.ServerAddress, buck
|
||||
Condition: cond,
|
||||
Mutations: []*filer_pb.ObjectMutation{
|
||||
s3a.latestPointerRecompute(bucket, object, isNewFormatVersionId(versionId), versionFileName, false),
|
||||
{Type: filer_pb.ObjectMutation_DELETE, Directory: versionsPath, Name: versionFileName, IsDeleteData: true, RemoveEmptyParent: true},
|
||||
{Type: filer_pb.ObjectMutation_DELETE, Directory: versionsPath, Name: versionFileName, IsDeleteData: true, RemoveEmptyParent: true, BypassGovernanceRetention: bypass},
|
||||
},
|
||||
}
|
||||
resp, err := s3a.objectTxnOnFiler(owner, req)
|
||||
@@ -202,7 +202,7 @@ func (s3a *S3ApiServer) routedDeleteSpecificVersion(owner pb.ServerAddress, buck
|
||||
case err != nil:
|
||||
glog.Errorf("routedDeleteSpecificVersion: %s/%s %s on %s: %v", bucket, object, versionId, owner, err)
|
||||
return s3err.ErrInternalError
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED || resp.ErrorCode == filer_pb.FilerError_ENTRY_LOCKED:
|
||||
// Legal hold or retention in force on the version.
|
||||
return s3err.ErrAccessDenied
|
||||
case resp.Error != "":
|
||||
@@ -227,14 +227,14 @@ func (s3a *S3ApiServer) routedDeleteNullVersion(owner pb.ServerAddress, bucket,
|
||||
RouteKey: s3a.objectRouteKey(bucket, object),
|
||||
Condition: wormDeleteCondition(worm, bypass),
|
||||
Mutations: []*filer_pb.ObjectMutation{
|
||||
{Type: filer_pb.ObjectMutation_DELETE, Directory: dir, Name: name, IsDeleteData: true},
|
||||
{Type: filer_pb.ObjectMutation_DELETE, Directory: dir, Name: name, IsDeleteData: true, BypassGovernanceRetention: bypass},
|
||||
},
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Warningf("routedDeleteNullVersion: %s/%s on %s, falling back to lock: %v", bucket, object, owner, err)
|
||||
return s3err.ErrNone, false
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED || resp.ErrorCode == filer_pb.FilerError_ENTRY_LOCKED:
|
||||
return s3err.ErrAccessDenied, true
|
||||
case resp.Error != "":
|
||||
glog.Warningf("routedDeleteNullVersion: %s/%s returned %q, falling back to lock", bucket, object, resp.Error)
|
||||
|
||||
@@ -1251,7 +1251,7 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(ctx context.Context, bucket,
|
||||
// File doesn't exist anymore, deletion was successful
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to delete null version %s: %v", normalizedObject, deleteErr)
|
||||
return fmt.Errorf("failed to delete null version %s: %w", normalizedObject, deleteErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1302,7 +1302,7 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(ctx context.Context, bucket,
|
||||
return nil
|
||||
}
|
||||
// File still exists but deletion failed for another reason
|
||||
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
|
||||
return fmt.Errorf("failed to delete version %s: %w", versionId, deleteErr)
|
||||
}
|
||||
|
||||
switch {
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
@@ -212,7 +213,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
|
||||
// Held apart from the named resp, which the local path below writes into:
|
||||
// a failed forward must not leave it nil.
|
||||
var ownerResp *filer_pb.CreateEntryResponse
|
||||
handled, forwardErr := fs.forwardToWriteOwner(ctx, entryRouteKey(fullpath), func(owner pb.ServerAddress) error {
|
||||
handled, forwardErr := fs.forwardToWriteOwner(ctx, entryRouteKey(fullpath), func(ctx context.Context, owner pb.ServerAddress) error {
|
||||
glog.V(2).InfofCtx(ctx, "CreateEntry %s: forwarding to owner %s", fullpath, owner)
|
||||
req.IsMoved = true
|
||||
return pb.WithFilerClient(false, 0, owner, fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
@@ -348,7 +349,7 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.Obje
|
||||
ConditionAltKey: req.ConditionAltKey,
|
||||
}
|
||||
var resp *filer_pb.ObjectTransactionResponse
|
||||
handled, err := fs.forwardToWriteOwner(ctx, req.RouteKey, func(owner pb.ServerAddress) error {
|
||||
handled, err := fs.forwardToWriteOwner(ctx, req.RouteKey, func(ctx context.Context, owner pb.ServerAddress) error {
|
||||
glog.V(2).InfofCtx(ctx, "ObjectTransaction %s: forwarding to owner %s", req.LockKey, owner)
|
||||
return pb.WithFilerClient(false, 0, owner, fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
var e error
|
||||
@@ -434,6 +435,9 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.Obje
|
||||
if err := fs.applyObjectMutation(ctx, m, req.IsFromOtherCluster, req.Signatures); err != nil {
|
||||
glog.V(2).InfofCtx(ctx, "ObjectTransaction %s mutation %d (%v): %v", lockPath, i, m.Type, err)
|
||||
resp.Error = fmt.Sprintf("mutation %d: %v", i, err)
|
||||
if errors.Is(err, filer.ErrEntryLocked) {
|
||||
resp.ErrorCode = filer_pb.FilerError_ENTRY_LOCKED
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
@@ -595,6 +599,16 @@ func (fs *FilerServer) applyStorageDefaultsToEntry(ctx context.Context, entry *f
|
||||
return so, nil
|
||||
}
|
||||
|
||||
// adminPeerSigningKey is nil on servers built without a credential guard
|
||||
// (tests, embedded filers), where bypass requests are refused the same way an
|
||||
// unkeyed deployment refuses non-local ones.
|
||||
func (fs *FilerServer) adminPeerSigningKey() security.SigningKey {
|
||||
if fs.filerGuard == nil {
|
||||
return nil
|
||||
}
|
||||
return fs.filerGuard.SigningKey()
|
||||
}
|
||||
|
||||
// applyObjectMutation applies a single mutation while the transaction's path
|
||||
// lock is held. PUT entries are expected to be fully prepared by the caller
|
||||
// (chunks resolved); mutations here are metadata-scoped. A DELETE of an absent
|
||||
@@ -618,6 +632,13 @@ func (fs *FilerServer) applyObjectMutation(ctx context.Context, m *filer_pb.Obje
|
||||
if m.KeepRemoteObject {
|
||||
ctx = filer.WithKeepRemoteObject(ctx)
|
||||
}
|
||||
if m.BypassGovernanceRetention {
|
||||
if peerErr := security.CheckGrpcAdminPeer(ctx, fs.adminPeerSigningKey()); peerErr == nil {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
} else {
|
||||
glog.V(1).InfofCtx(ctx, "ObjectMutation DELETE %s/%s: bypass_governance_retention ignored: %v", m.Directory, m.Name, peerErr)
|
||||
}
|
||||
}
|
||||
err := fs.filer.DeleteEntryMetaAndData(ctx, fullpath, m.IsRecursive, false, m.IsDeleteData, fromOtherCluster, signatures, 0)
|
||||
if err != nil && err != filer_pb.ErrNotFound {
|
||||
return err
|
||||
@@ -829,7 +850,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
|
||||
routed := conditionIsSet(req.Condition) || len(req.ExpectedExtended) > 0
|
||||
if !req.IsMoved && routed {
|
||||
var ownerResp *filer_pb.UpdateEntryResponse
|
||||
handled, forwardErr := fs.forwardToWriteOwner(ctx, entryRouteKey(util.FullPath(fullpath)), func(owner pb.ServerAddress) error {
|
||||
handled, forwardErr := fs.forwardToWriteOwner(ctx, entryRouteKey(util.FullPath(fullpath)), func(ctx context.Context, owner pb.ServerAddress) error {
|
||||
glog.V(2).InfofCtx(ctx, "UpdateEntry %s: forwarding to owner %s", fullpath, owner)
|
||||
req.IsMoved = true
|
||||
return pb.WithFilerClient(false, 0, owner, fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
@@ -1052,6 +1073,13 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
|
||||
if req.KeepRemoteObject {
|
||||
ctx = filer.WithKeepRemoteObject(ctx)
|
||||
}
|
||||
if req.BypassGovernanceRetention {
|
||||
if peerErr := security.CheckGrpcAdminPeer(ctx, fs.adminPeerSigningKey()); peerErr == nil {
|
||||
ctx = filer.WithBypassGovernanceRetention(ctx)
|
||||
} else {
|
||||
glog.V(1).InfofCtx(ctx, "DeleteEntry %s: bypass_governance_retention ignored: %v", fullpath, peerErr)
|
||||
}
|
||||
}
|
||||
ctx, eventSink := filer.WithMetadataEventSink(ctx)
|
||||
err = fs.filer.DeleteEntryMetaAndData(ctx, fullpath, req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures, req.IfNotModifiedAfter)
|
||||
resp = &filer_pb.DeleteEntryResponse{}
|
||||
|
||||
@@ -1095,3 +1095,32 @@ func TestObjectTransactionConditionIgnoresDirectory(t *testing.T) {
|
||||
t.Fatalf("directory at key with live latest: err=%v code=%v, want precondition failed", err, resp.GetErrorCode())
|
||||
}
|
||||
}
|
||||
|
||||
// A mutation-level lock refusal reports ENTRY_LOCKED: the "mutation N:" prefix
|
||||
// on resp.Error strips the leading marker IsEntryLockedError reads, so the
|
||||
// structured code is what a routed delete maps to AccessDenied.
|
||||
func TestObjectTransactionDeleteLockedEntry(t *testing.T) {
|
||||
now := time.Unix(1700000000, 0)
|
||||
fs, store := newTxnTestServer(t, map[string]*filer.Entry{
|
||||
"/buckets/b/obj": {
|
||||
Attr: filer.Attr{Inode: 1, Mtime: now, Crtime: now, Mode: 0644},
|
||||
Extended: map[string][]byte{s3_constants.ExtLegalHoldKey: []byte("on")},
|
||||
},
|
||||
})
|
||||
|
||||
resp, err := fs.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{
|
||||
LockKey: "/buckets/b/obj",
|
||||
Mutations: []*filer_pb.ObjectMutation{
|
||||
{Type: filer_pb.ObjectMutation_DELETE, Directory: "/buckets/b", Name: "obj"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if resp.ErrorCode != filer_pb.FilerError_ENTRY_LOCKED {
|
||||
t.Fatalf("error code = %v, want ENTRY_LOCKED (error %q)", resp.ErrorCode, resp.Error)
|
||||
}
|
||||
if _, ok := store.entries["/buckets/b/obj"]; !ok {
|
||||
t.Errorf("locked entry should survive the refused delete")
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/peer"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
@@ -44,18 +45,30 @@ func (fs *FilerServer) writeOwner(key string) pb.ServerAddress {
|
||||
// already committed — and an owner unreachable from here may be partitioned
|
||||
// rather than down, still serving the key to everyone else. The ring hands the
|
||||
// key on when the cooling-off window closes, so the outage is bounded.
|
||||
func (fs *FilerServer) forwardToWriteOwner(ctx context.Context, key string, send func(owner pb.ServerAddress) error) (handled bool, err error) {
|
||||
func (fs *FilerServer) forwardToWriteOwner(ctx context.Context, key string, send func(ctx context.Context, owner pb.ServerAddress) error) (handled bool, err error) {
|
||||
owner := fs.writeOwner(key)
|
||||
if owner == "" {
|
||||
return false, nil
|
||||
}
|
||||
if err := send(owner); err != nil {
|
||||
if err := send(forwardedAuthContext(ctx), owner); err != nil {
|
||||
glog.V(1).InfofCtx(ctx, "route %s to owner %s: %v", key, owner, err)
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// forwardedAuthContext carries the caller's authorization metadata onto the
|
||||
// owner hop so checks like the governance-bypass admin-peer check see the
|
||||
// same credential the entry filer did.
|
||||
func forwardedAuthContext(ctx context.Context) context.Context {
|
||||
if md, ok := metadata.FromIncomingContext(ctx); ok {
|
||||
if auth := md.Get("authorization"); len(auth) > 0 {
|
||||
return metadata.AppendToOutgoingContext(ctx, "authorization", auth[0])
|
||||
}
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// entryRouteKey is the ring key for an entry's writes. It shares the S3
|
||||
// gateway's namespace so an object's ObjectTransaction and its CreateEntry
|
||||
// resolve to the same owner, and land on that filer's one per-path lock.
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestForwardToWriteOwnerAppliesLocallyWhenOwned(t *testing.T) {
|
||||
ring := fs.filer.Dlm.LockRing
|
||||
|
||||
key := firstKey(t, func(key string) bool { return ring.GetPrimary(key) == "f2:8888" })
|
||||
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(pb.ServerAddress) error {
|
||||
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(context.Context, pb.ServerAddress) error {
|
||||
t.Fatal("must not forward a key this filer owns")
|
||||
return nil
|
||||
})
|
||||
@@ -95,7 +95,7 @@ func TestForwardToWriteOwnerNeverTriesASecondFiler(t *testing.T) {
|
||||
|
||||
moved := firstKey(t, func(key string) bool { return ring.PriorOwner(key) != "" })
|
||||
var tried []pb.ServerAddress
|
||||
handled, err := fs.forwardToWriteOwner(context.Background(), moved, func(owner pb.ServerAddress) error {
|
||||
handled, err := fs.forwardToWriteOwner(context.Background(), moved, func(ctx context.Context, owner pb.ServerAddress) error {
|
||||
tried = append(tried, owner)
|
||||
return errors.New("dial tcp: connection refused")
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user