From df93d01c0685354a9ffcf092d8effc31c9aebadf Mon Sep 17 00:00:00 2001 From: Mathieu Arnold Date: Sat, 22 Aug 2026 08:42:26 +0200 Subject: [PATCH] admin: add bucket lifecycle rule editing (#10860) * admin: add bucket lifecycle rule editing * address greptile's comments * more small fixes * coderabbit's comments * more comment fixes * more fixes * more * maybe last * last ? * 14850 * 14851 * filer: stamp the content MD5 on every SaveInsideFiler write An entry's ETag falls back to Attributes.Md5, so conditional writers key IF_ETAG_MATCH off it. SaveInsideFiler carried the looked-up attributes forward without refreshing the hash, leaving it describing whatever the previous writer stored: a later conditional write matched the stale hash and overwrote content that had already changed. * s3api: give the bucket lifecycle constants and the write route key one definition each The extended-attribute keys, the XML size cap and the object-write ring key prefix were each spelled out in two places, so the admin dashboard's copies could drift from the gateway's. Move them to the packages both sides already import and alias them where the short local name reads better. * admin: patch the bucket entry's lifecycle keys instead of rewriting the entry The save read the bucket entry, edited its extended map and wrote the whole entry back, guarded by IF_UNMODIFIED_SINCE. Nothing that writes a bucket entry advances its mtime - not the S3 gateway's patchBucketEntry, not SetBucketOwner, not SetBucketQuota - so the guard never fired and the stale snapshot reverted whatever else had changed since the lookup. Send the PATCH_EXTENDED mutation the S3 gateway already uses for these keys: the filer re-reads and merges under the bucket path lock, so only the two lifecycle keys move. That removes the reason for the mtime snapshot, the verification retry loop and the compensating restore of the cleared day-TTL rules, which the migration now logs instead. * s3api: run the delete-lifecycle day-TTL migration through the shared helper DeleteBucketLifecycleHandler kept its own copy of the read-strip-write sequence the put handler now shares, including a missing return that let a ToText failure persist a truncated filer.conf and write a second response. It also wrote the whole file back unconditionally, reverting any concurrent edit; the shared helper writes conditionally. * admin: answer 404 when a lifecycle request names a bucket that does not exist Every SetBucketLifecycle failure came back as 500, including the lookup miss for an unknown bucket, so a client or monitor read a caller error as a server fault and retried it. * s3api: emit lifecycle XML a client would recognize Two changes to what MarshalCanonical writes, both visible through GetBucketLifecycleConfiguration, which replays the stored bytes verbatim: stamp the S3 namespace on the root, and put a size range under . A carries one predicate, so two size bounds side by side is a shape AWS does not document. Parsing still accepts either. * admin: fix the lifecycle editor's handling of stored status, deletes and empty saves Four things the editor got wrong: A stored the S3 API never validated, say 'enabled', left both radio buttons unchecked, so reading the form threw on a null querySelector result and Save did nothing. Collapse anything but an exact 'Enabled' to 'Disabled', which is what the engine already does with it. Deleting a rule re-rendered an open edit form from the snapshot taken when editing began, discarding what had been typed; every other transition folds the form in first. The Transition warning only matched a bare , missing the form with attributes, self-closed or namespace-prefixed. Saving an emptied rule list clears the configuration through a path with no prompt, next to a Delete-all-rules button that asks. Also collapses the three divergent copies of formatBytes on this page to one. * filer: stop the day-TTL migration from deleting an operator's path rule The migration removed every rule under the bucket's path that carried a day TTL in the bucket's collection. The add path it is retiring used AddLocationConf, which merged its TTL onto whatever already sat at the prefix, so a rule can hold operator settings the lifecycle path never wrote - a disk type, WORM retention, a read-only flag, a placement pin. Deleting the whole rule to retire its TTL took those with it, leaving objects under that prefix on defaults nobody asked for. Delete only rules shaped like ones the add path created from scratch; anything else keeps its settings and loses just the TTL. --------- Co-authored-by: Chris Lu Co-authored-by: Chris Lu --- weed/admin/dash/admin_server.go | 127 +++ weed/admin/dash/bucket_lifecycle_test.go | 350 ++++++++ weed/admin/dash/bucket_management.go | 155 ++++ weed/admin/handlers/admin_handlers.go | 2 + .../handlers/admin_handlers_routes_test.go | 20 + weed/admin/view/app/s3_buckets.templ | 759 +++++++++++++++--- weed/admin/view/app/s3_buckets_templ.go | 2 +- weed/filer/filer_conf.go | 171 ++++ weed/filer/filer_conf_test.go | 341 ++++++++ weed/filer/read_write.go | 10 + weed/s3api/lifecycle_xml/canonical.go | 141 ++++ weed/s3api/lifecycle_xml/round_trip_test.go | 323 ++++++++ weed/s3api/s3_constants/buckets.go | 5 + weed/s3api/s3api_bucket_handlers.go | 65 +- weed/s3api/s3api_bucket_lifecycle_config.go | 10 +- weed/s3api/s3api_object_routed_write.go | 4 +- .../s3api/s3lifecycle/scheduler/configload.go | 13 +- 17 files changed, 2313 insertions(+), 185 deletions(-) create mode 100644 weed/admin/dash/bucket_lifecycle_test.go diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 137a976f0..37f89de83 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -1001,6 +1001,133 @@ func toBucketLifecycleRule(rule *s3lifecycle.Rule) BucketLifecycleRule { return out } +// fromBucketLifecycleRule is the inverse of toBucketLifecycleRule, turning a +// rule edited in the admin UI back into the engine's canonical shape. +func fromBucketLifecycleRule(rule BucketLifecycleRule) (*s3lifecycle.Rule, error) { + out := &s3lifecycle.Rule{ + ID: rule.ID, + Status: rule.Status, + Prefix: rule.Prefix, + FilterTags: rule.Tags, + FilterSizeGreaterThan: rule.SizeGreaterThan, + FilterSizeLessThan: rule.SizeLessThan, + ExpirationDays: rule.ExpirationDays, + ExpiredObjectDeleteMarker: rule.ExpiredObjectDeleteMarker, + NoncurrentVersionExpirationDays: rule.NoncurrentVersionExpirationDays, + NewerNoncurrentVersions: rule.NewerNoncurrentVersions, + AbortMPUDaysAfterInitiation: rule.AbortMultipartDays, + } + if rule.ExpirationDate != "" { + date, err := time.Parse(time.DateOnly, rule.ExpirationDate) + if err != nil { + return nil, fmt.Errorf("invalid expiration date %q: %w", rule.ExpirationDate, err) + } + out.ExpirationDate = date + } + return out, nil +} + +// ErrBucketNotFound reports that the named bucket has no filer entry, so a +// handler can answer 404 rather than 500. +var ErrBucketNotFound = errors.New("bucket not found") + +// SetBucketLifecycle replaces the lifecycle configuration stored on a +// bucket's filer entry. An empty rule list clears the configuration +// entirely, mirroring clearStoredBucketLifecycleConfiguration on the S3 API +// side. Callers must validate rules before calling this (see +// validateBucketLifecycleRules) — this only rejects what marshaling itself +// rejects. +func (s *AdminServer) SetBucketLifecycle(bucketName string, rules []BucketLifecycleRule) error { + canonicalRules := make([]*s3lifecycle.Rule, 0, len(rules)) + for _, rule := range rules { + canonicalRule, err := fromBucketLifecycleRule(rule) + if err != nil { + return err + } + canonicalRules = append(canonicalRules, canonicalRule) + } + + var lifecycleXML []byte + if len(canonicalRules) > 0 { + var err error + lifecycleXML, err = lifecycle_xml.MarshalCanonical(canonicalRules) + if err != nil { + return fmt.Errorf("marshal lifecycle configuration: %w", err) + } + if len(lifecycleXML) > scheduler.MaxBucketLifecycleConfigurationSize { + return fmt.Errorf("lifecycle configuration is %d bytes, which exceeds the %d byte limit", len(lifecycleXML), scheduler.MaxBucketLifecycleConfigurationSize) + } + } + + filerConfig, err := s.getFilerConfig() + if err != nil { + return fmt.Errorf("get filer configuration: %w", err) + } + collection := getCollectionName(filerConfig.FilerGroup, bucketName) + + return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + // PATCH_EXTENDED is a no-op on a missing entry, so the existence + // check has to happen here rather than fall out of the write. + if _, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: filerConfig.BucketsPath, + Name: bucketName, + }); err != nil { + if errors.Is(err, filer_pb.ErrNotFound) { + return fmt.Errorf("%w: %s", ErrBucketNotFound, bucketName) + } + return fmt.Errorf("look up bucket %s: %w", bucketName, err) + } + + // Migration: clear any legacy day-TTL filer.conf entries before + // writing the new XML, so a failure here leaves the bucket entry + // untouched instead of committing the new policy alongside a stale + // TTL rule. Same step and ordering as + // PutBucketLifecycleConfigurationHandler. + if err := filer.ClearBucketLifecycleDayTTLs(context.Background(), client, filerConfig.BucketsPath, bucketName, collection); err != nil { + return fmt.Errorf("failed to clear legacy lifecycle TTLs: %w", err) + } + + bucketPath := filerConfig.BucketsPath + "/" + bucketName + resp, err := client.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{ + LockKey: bucketPath, + RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + bucketPath, + Mutations: []*filer_pb.ObjectMutation{bucketLifecycleMutation(filerConfig.BucketsPath, bucketName, lifecycleXML)}, + }) + if err != nil { + return fmt.Errorf("failed to update bucket lifecycle: %w", err) + } + if resp.Error != "" { + return fmt.Errorf("failed to update bucket lifecycle: %s", resp.Error) + } + return nil + }) +} + +// bucketLifecycleMutation patches the two lifecycle keys rather than writing +// the whole entry back: the filer re-reads and merges under the bucket path +// lock, so a concurrent owner/quota/versioning change is preserved instead of +// being reverted by a stale snapshot. Same mutation the S3 gateway uses for +// these keys (see patchBucketEntry in s3api_bucket_config.go). Empty XML +// clears the configuration, transition minimum size included. +func bucketLifecycleMutation(bucketsPath, bucketName string, lifecycleXML []byte) *filer_pb.ObjectMutation { + mutation := &filer_pb.ObjectMutation{ + Type: filer_pb.ObjectMutation_PATCH_EXTENDED, + Directory: bucketsPath, + Name: bucketName, + } + if len(lifecycleXML) > 0 { + mutation.SetExtended = map[string][]byte{ + scheduler.BucketLifecycleConfigurationXMLKey: lifecycleXML, + } + return mutation + } + mutation.DeleteExtended = []string{ + scheduler.BucketLifecycleConfigurationXMLKey, + scheduler.BucketLifecycleTransitionMinimumObjectSizeKey, + } + return mutation +} + // CreateS3Bucket creates a new S3 bucket func (s *AdminServer) CreateS3Bucket(bucketName string) error { return s.CreateS3BucketWithQuota(bucketName, 0, false) diff --git a/weed/admin/dash/bucket_lifecycle_test.go b/weed/admin/dash/bucket_lifecycle_test.go new file mode 100644 index 000000000..63cc9c1cb --- /dev/null +++ b/weed/admin/dash/bucket_lifecycle_test.go @@ -0,0 +1,350 @@ +package dash + +import ( + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler" +) + +func minimalLifecycleRule() BucketLifecycleRule { + return BucketLifecycleRule{ + ID: "rule-1", + Status: s3lifecycle.StatusEnabled, + ExpirationDays: 30, + } +} + +func TestValidateBucketLifecycleRules_Valid(t *testing.T) { + if err := validateBucketLifecycleRules([]BucketLifecycleRule{minimalLifecycleRule()}); err != nil { + t.Fatalf("expected valid rule set to pass, got: %v", err) + } +} + +func TestValidateBucketLifecycleRules_Empty(t *testing.T) { + if err := validateBucketLifecycleRules(nil); err != nil { + t.Fatalf("expected empty rule set to pass, got: %v", err) + } +} + +func TestValidateBucketLifecycleRules_TooManyRules(t *testing.T) { + rules := make([]BucketLifecycleRule, MaxBucketLifecycleRules+1) + for i := range rules { + rules[i] = BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, ExpirationDays: 30} + } + if err := validateBucketLifecycleRules(rules); err == nil { + t.Fatal("expected error for exceeding the rule count cap") + } +} + +func TestValidateBucketLifecycleRules_DuplicateIDs(t *testing.T) { + rule := minimalLifecycleRule() + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule, rule}); err == nil { + t.Fatal("expected error for duplicate rule IDs") + } +} + +func TestValidateBucketLifecycleRules_IDTooLong(t *testing.T) { + rule := minimalLifecycleRule() + longID := "" + for i := 0; i < MaxBucketLifecycleRuleIDLength+1; i++ { + longID += "a" + } + rule.ID = longID + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for an ID longer than the cap") + } +} + +func TestValidateBucketLifecycleRules_InvalidStatus(t *testing.T) { + rule := minimalLifecycleRule() + rule.Status = "sort-of-enabled" + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for an invalid status") + } +} + +func TestValidateBucketLifecycleRules_ExpirationDaysAndDateMutuallyExclusive(t *testing.T) { + rule := minimalLifecycleRule() + rule.ExpirationDate = "2030-01-01" + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error when both expiration_days and expiration_date are set") + } +} + +func TestValidateBucketLifecycleRules_DeleteMarkerWithExpirationDaysRejected(t *testing.T) { + // AWS forbids combining ExpiredObjectDeleteMarker with Days/Date in the + // same Expiration element; ruleFromCanonical would otherwise emit both + // on the same . + rule := minimalLifecycleRule() + rule.ExpiredObjectDeleteMarker = true + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error when expired_object_delete_marker is combined with expiration_days") + } +} + +func TestValidateBucketLifecycleRules_DeleteMarkerWithExpirationDateRejected(t *testing.T) { + rule := BucketLifecycleRule{ + Status: s3lifecycle.StatusEnabled, + ExpirationDate: "2030-01-01", + ExpiredObjectDeleteMarker: true, + } + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error when expired_object_delete_marker is combined with expiration_date") + } +} + +func TestValidateBucketLifecycleRules_DeleteMarkerAlone(t *testing.T) { + rule := BucketLifecycleRule{ + Status: s3lifecycle.StatusEnabled, + ExpiredObjectDeleteMarker: true, + } + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err != nil { + t.Fatalf("expected standalone expired_object_delete_marker to be accepted, got: %v", err) + } +} + +func TestValidateBucketLifecycleRules_InvalidExpirationDate(t *testing.T) { + rule := BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, ExpirationDate: "01/01/2030"} + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for a malformed expiration_date") + } +} + +func TestValidateBucketLifecycleRules_NegativeExpirationDays(t *testing.T) { + rule := minimalLifecycleRule() + rule.ExpirationDays = -1 + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for negative expiration_days") + } +} + +// TestValidateBucketLifecycleRules_StandaloneNewerNoncurrentVersions guards +// against re-adding a NoncurrentVersionExpirationDays requirement: +// s3lifecycle.RuleActionKinds recognizes a standalone NewerNoncurrentVersions +// (no days) as ActionKindNewerNoncurrent, a valid count-only retention rule +// the S3 API already accepts and stores. +func TestValidateBucketLifecycleRules_StandaloneNewerNoncurrentVersions(t *testing.T) { + rule := BucketLifecycleRule{ + Status: s3lifecycle.StatusEnabled, + NewerNoncurrentVersions: 2, + } + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err != nil { + t.Fatalf("expected standalone newer_noncurrent_versions to be accepted, got: %v", err) + } +} + +func TestValidateBucketLifecycleRules_NegativeAbortMultipartDays(t *testing.T) { + rule := minimalLifecycleRule() + rule.AbortMultipartDays = -5 + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for negative abort_multipart_days") + } +} + +func TestValidateBucketLifecycleRules_SizeGreaterThanNotLessThanSizeLessThan(t *testing.T) { + rule := minimalLifecycleRule() + rule.SizeGreaterThan = 2048 + rule.SizeLessThan = 1024 + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error when size_greater_than >= size_less_than") + } +} + +func TestValidateBucketLifecycleRules_NegativeSizeBounds(t *testing.T) { + rule := minimalLifecycleRule() + rule.SizeGreaterThan = -1 + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for a negative size_greater_than") + } +} + +func TestValidateBucketLifecycleRules_EmptyTagKey(t *testing.T) { + rule := minimalLifecycleRule() + rule.Tags = map[string]string{"": "value"} + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for an empty tag key") + } +} + +func TestValidateBucketLifecycleRules_EmptyTagValue(t *testing.T) { + rule := minimalLifecycleRule() + rule.Tags = map[string]string{"env": " "} + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for a blank tag value") + } +} + +func TestValidateBucketLifecycleRules_NoAction(t *testing.T) { + rule := BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, Prefix: "logs/"} + if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil { + t.Fatal("expected error for a rule with no action") + } +} + +func TestFromBucketLifecycleRule_MapsAllFields(t *testing.T) { + rule := BucketLifecycleRule{ + ID: "rule-1", + Status: s3lifecycle.StatusEnabled, + Prefix: "logs/", + Tags: map[string]string{"env": "dev"}, + SizeGreaterThan: 1024, + SizeLessThan: 2048, + ExpirationDate: "2030-01-15", + ExpiredObjectDeleteMarker: true, + NoncurrentVersionExpirationDays: 30, + NewerNoncurrentVersions: 2, + AbortMultipartDays: 7, + } + + out, err := fromBucketLifecycleRule(rule) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if out.ID != rule.ID { + t.Errorf("ID = %q, want %q", out.ID, rule.ID) + } + if out.Status != rule.Status { + t.Errorf("Status = %q, want %q", out.Status, rule.Status) + } + if out.Prefix != rule.Prefix { + t.Errorf("Prefix = %q, want %q", out.Prefix, rule.Prefix) + } + if len(out.FilterTags) != 1 || out.FilterTags["env"] != "dev" { + t.Errorf("FilterTags = %v, want {env: dev}", out.FilterTags) + } + if out.FilterSizeGreaterThan != rule.SizeGreaterThan { + t.Errorf("FilterSizeGreaterThan = %d, want %d", out.FilterSizeGreaterThan, rule.SizeGreaterThan) + } + if out.FilterSizeLessThan != rule.SizeLessThan { + t.Errorf("FilterSizeLessThan = %d, want %d", out.FilterSizeLessThan, rule.SizeLessThan) + } + wantDate, _ := time.Parse(time.DateOnly, rule.ExpirationDate) + if !out.ExpirationDate.Equal(wantDate) { + t.Errorf("ExpirationDate = %v, want %v", out.ExpirationDate, wantDate) + } + if !out.ExpiredObjectDeleteMarker { + t.Error("expected ExpiredObjectDeleteMarker=true") + } + if out.NoncurrentVersionExpirationDays != rule.NoncurrentVersionExpirationDays { + t.Errorf("NoncurrentVersionExpirationDays = %d, want %d", out.NoncurrentVersionExpirationDays, rule.NoncurrentVersionExpirationDays) + } + if out.NewerNoncurrentVersions != rule.NewerNoncurrentVersions { + t.Errorf("NewerNoncurrentVersions = %d, want %d", out.NewerNoncurrentVersions, rule.NewerNoncurrentVersions) + } + if out.AbortMPUDaysAfterInitiation != rule.AbortMultipartDays { + t.Errorf("AbortMPUDaysAfterInitiation = %d, want %d", out.AbortMPUDaysAfterInitiation, rule.AbortMultipartDays) + } +} + +func TestFromBucketLifecycleRule_EmptyExpirationDate(t *testing.T) { + rule := minimalLifecycleRule() + rule.ExpirationDate = "" + + out, err := fromBucketLifecycleRule(rule) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !out.ExpirationDate.IsZero() { + t.Errorf("expected zero ExpirationDate, got %v", out.ExpirationDate) + } +} + +func TestFromBucketLifecycleRule_InvalidExpirationDate(t *testing.T) { + rule := minimalLifecycleRule() + rule.ExpirationDate = "not-a-date" + + if _, err := fromBucketLifecycleRule(rule); err == nil { + t.Fatal("expected error for a malformed expiration date") + } +} + +func TestBucketLifecycleMutation_SetsXML(t *testing.T) { + m := bucketLifecycleMutation("/buckets", "mybucket", []byte("")) + + if m.Type != filer_pb.ObjectMutation_PATCH_EXTENDED { + t.Fatalf("expected a PATCH_EXTENDED mutation, got %v", m.Type) + } + if m.Directory != "/buckets" || m.Name != "mybucket" { + t.Fatalf("expected the mutation to target /buckets/mybucket, got %s/%s", m.Directory, m.Name) + } + if got := string(m.SetExtended[scheduler.BucketLifecycleConfigurationXMLKey]); got != "" { + t.Fatalf("expected the XML key to carry the marshaled config, got %q", got) + } + if len(m.DeleteExtended) != 0 { + t.Fatalf("expected no key deletions when saving rules, got %v", m.DeleteExtended) + } +} + +func TestBucketLifecycleMutation_ClearsBothKeys(t *testing.T) { + m := bucketLifecycleMutation("/buckets", "mybucket", nil) + + if len(m.SetExtended) != 0 { + t.Fatalf("expected no key writes when clearing, got %v", m.SetExtended) + } + want := map[string]bool{ + scheduler.BucketLifecycleConfigurationXMLKey: true, + scheduler.BucketLifecycleTransitionMinimumObjectSizeKey: true, + } + if len(m.DeleteExtended) != len(want) { + t.Fatalf("expected both lifecycle keys to be cleared, got %v", m.DeleteExtended) + } + for _, k := range m.DeleteExtended { + if !want[k] { + t.Fatalf("unexpected key cleared: %s", k) + } + } +} + +// A whole-entry write would have carried the rest of the bucket entry with it; +// the patch must name only the keys it owns, so a concurrent owner or quota +// change survives. +func TestBucketLifecycleMutation_TouchesOnlyLifecycleKeys(t *testing.T) { + for _, m := range []*filer_pb.ObjectMutation{ + bucketLifecycleMutation("/buckets", "mybucket", []byte("")), + bucketLifecycleMutation("/buckets", "mybucket", nil), + } { + if m.Entry != nil { + t.Fatal("expected the mutation to carry no entry snapshot") + } + if m.SetContent { + t.Fatal("expected the mutation to leave entry content alone") + } + for k := range m.SetExtended { + if k != scheduler.BucketLifecycleConfigurationXMLKey { + t.Fatalf("unexpected key written: %s", k) + } + } + } +} + +func TestSetBucketLifecycle_RejectsOversizedConfiguration(t *testing.T) { + // A single rule ID long enough to blow the cap: this must fail before any + // filer call, which is what makes it testable without one. + rule := minimalLifecycleRule() + rule.ID = strings.Repeat("x", scheduler.MaxBucketLifecycleConfigurationSize+1) + + err := (&AdminServer{}).SetBucketLifecycle("mybucket", []BucketLifecycleRule{rule}) + if err == nil { + t.Fatal("expected an oversized lifecycle configuration to be rejected") + } + if !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("expected a size-limit error, got: %v", err) + } +} + +func TestBucketLifecycleErrorStatus(t *testing.T) { + if got := bucketLifecycleErrorStatus(fmt.Errorf("%w: mybucket", ErrBucketNotFound)); got != http.StatusNotFound { + t.Fatalf("expected a missing bucket to map to 404, got %d", got) + } + if got := bucketLifecycleErrorStatus(errors.New("filer unreachable")); got != http.StatusInternalServerError { + t.Fatalf("expected an unrelated failure to stay 500, got %d", got) + } +} diff --git a/weed/admin/dash/bucket_management.go b/weed/admin/dash/bucket_management.go index b68d764bf..2e6072e52 100644 --- a/weed/admin/dash/bucket_management.go +++ b/weed/admin/dash/bucket_management.go @@ -2,6 +2,7 @@ package dash import ( "context" + "errors" "fmt" "net/http" "os" @@ -13,8 +14,15 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" ) +// MaxBucketLifecycleRules mirrors AWS S3's limit of 1000 lifecycle rules per bucket. +const MaxBucketLifecycleRules = 1000 + +// MaxBucketLifecycleRuleIDLength mirrors the S3 API's limit on a lifecycle rule's . +const MaxBucketLifecycleRuleIDLength = 255 + // MaxOwnerNameLength is the maximum allowed length for bucket owner identity names. // This is a reasonable limit to prevent abuse; AWS IAM user names are limited to 64 chars, // but we use 256 to allow for more complex identity formats (e.g., email addresses). @@ -102,6 +110,153 @@ func (s *AdminServer) ShowBucketLifecycle(w http.ResponseWriter, r *http.Request writeJSON(w, http.StatusOK, lifecycle) } +// UpdateBucketLifecycle replaces the entire lifecycle configuration for a bucket. +func (s *AdminServer) UpdateBucketLifecycle(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + + bucketName := mux.Vars(r)["bucket"] + if bucketName == "" { + writeJSONError(w, http.StatusBadRequest, "Bucket name is required") + return + } + + var req struct { + Rules []BucketLifecycleRule `json:"rules"` + } + if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { + writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) + return + } + + if err := validateBucketLifecycleRules(req.Rules); err != nil { + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + + if err := s.SetBucketLifecycle(bucketName, req.Rules); err != nil { + writeJSONError(w, bucketLifecycleErrorStatus(err), "Failed to update bucket lifecycle: "+err.Error()) + return + } + + writeJSON(w, http.StatusOK, map[string]interface{}{ + "message": "Bucket lifecycle updated successfully", + "bucket": bucketName, + }) +} + +// DeleteBucketLifecycle clears the entire lifecycle configuration for a bucket. +func (s *AdminServer) DeleteBucketLifecycle(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + + bucketName := mux.Vars(r)["bucket"] + if bucketName == "" { + writeJSONError(w, http.StatusBadRequest, "Bucket name is required") + return + } + + if err := s.SetBucketLifecycle(bucketName, nil); err != nil { + writeJSONError(w, bucketLifecycleErrorStatus(err), "Failed to delete bucket lifecycle: "+err.Error()) + return + } + + writeJSON(w, http.StatusOK, map[string]interface{}{ + "message": "Bucket lifecycle deleted successfully", + "bucket": bucketName, + }) +} + +// bucketLifecycleErrorStatus keeps a request for a bucket that does not exist +// out of the 5xx bucket, where a client would retry it. +func bucketLifecycleErrorStatus(err error) int { + if errors.Is(err, ErrBucketNotFound) { + return http.StatusNotFound + } + return http.StatusInternalServerError +} + +// validateBucketLifecycleRules rejects a rule set before any write is +// attempted, so a malformed PUT can't half-apply. Mirrors the constraints +// AWS enforces on PutBucketLifecycleConfiguration plus the local rule cap. +func validateBucketLifecycleRules(rules []BucketLifecycleRule) error { + if len(rules) > MaxBucketLifecycleRules { + return fmt.Errorf("a bucket may have at most %d lifecycle rules, got %d", MaxBucketLifecycleRules, len(rules)) + } + + seenIDs := make(map[string]bool, len(rules)) + for i, rule := range rules { + label := fmt.Sprintf("rule %d", i+1) + if rule.ID != "" { + label = fmt.Sprintf("rule %q", rule.ID) + if len(rule.ID) > MaxBucketLifecycleRuleIDLength { + return fmt.Errorf("%s: ID must be %d characters or less", label, MaxBucketLifecycleRuleIDLength) + } + if seenIDs[rule.ID] { + return fmt.Errorf("duplicate rule ID %q", rule.ID) + } + seenIDs[rule.ID] = true + } + + switch rule.Status { + case s3lifecycle.StatusEnabled, s3lifecycle.StatusDisabled: + default: + return fmt.Errorf("%s: status must be %q or %q, got %q", label, s3lifecycle.StatusEnabled, s3lifecycle.StatusDisabled, rule.Status) + } + + if rule.ExpirationDays > 0 && rule.ExpirationDate != "" { + return fmt.Errorf("%s: expiration_days and expiration_date are mutually exclusive", label) + } + if rule.ExpirationDays < 0 { + return fmt.Errorf("%s: expiration_days must be positive", label) + } + if rule.ExpirationDate != "" { + if _, err := time.Parse(time.DateOnly, rule.ExpirationDate); err != nil { + return fmt.Errorf("%s: invalid expiration_date %q, expected YYYY-MM-DD", label, rule.ExpirationDate) + } + } + if rule.ExpiredObjectDeleteMarker && (rule.ExpirationDays > 0 || rule.ExpirationDate != "") { + return fmt.Errorf("%s: expired_object_delete_marker cannot be combined with expiration_days or expiration_date", label) + } + + if rule.NoncurrentVersionExpirationDays < 0 { + return fmt.Errorf("%s: noncurrent_version_expiration_days must be positive", label) + } + if rule.NewerNoncurrentVersions < 0 { + return fmt.Errorf("%s: newer_noncurrent_versions must be positive", label) + } + if rule.AbortMultipartDays < 0 { + return fmt.Errorf("%s: abort_multipart_days must be positive", label) + } + + if rule.SizeGreaterThan < 0 || rule.SizeLessThan < 0 { + return fmt.Errorf("%s: size_greater_than and size_less_than must be positive", label) + } + if rule.SizeGreaterThan > 0 && rule.SizeLessThan > 0 && rule.SizeGreaterThan >= rule.SizeLessThan { + return fmt.Errorf("%s: size_greater_than must be less than size_less_than", label) + } + + for k, v := range rule.Tags { + if strings.TrimSpace(k) == "" { + return fmt.Errorf("%s: tag keys must not be empty", label) + } + if strings.TrimSpace(v) == "" { + return fmt.Errorf("%s: tag value for key %q must not be empty", label, k) + } + } + + hasAction := rule.ExpirationDays > 0 || rule.ExpirationDate != "" || rule.ExpiredObjectDeleteMarker || + rule.NoncurrentVersionExpirationDays > 0 || rule.NewerNoncurrentVersions > 0 || rule.AbortMultipartDays > 0 + if !hasAction { + return fmt.Errorf("%s: must specify at least one action (expiration, noncurrent version expiration, or abort incomplete multipart upload)", label) + } + } + + return nil +} + // CreateBucket creates a new S3 bucket func (s *AdminServer) CreateBucket(w http.ResponseWriter, r *http.Request) { var req CreateBucketRequest diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 85095ef37..097642407 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -179,6 +179,8 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { s3Api.Handle("/buckets/{bucket}", wrapWrite(h.adminServer.DeleteBucket)).Methods(http.MethodDelete) s3Api.HandleFunc("/buckets/{bucket}", h.adminServer.ShowBucketDetails).Methods(http.MethodGet) s3Api.HandleFunc("/buckets/{bucket}/lifecycle", h.adminServer.ShowBucketLifecycle).Methods(http.MethodGet) + s3Api.Handle("/buckets/{bucket}/lifecycle", wrapWrite(h.adminServer.UpdateBucketLifecycle)).Methods(http.MethodPut) + s3Api.Handle("/buckets/{bucket}/lifecycle", wrapWrite(h.adminServer.DeleteBucketLifecycle)).Methods(http.MethodDelete) s3Api.Handle("/buckets/{bucket}/quota", wrapWrite(h.adminServer.UpdateBucketQuota)).Methods(http.MethodPut) s3Api.Handle("/buckets/{bucket}/owner", wrapWrite(h.adminServer.UpdateBucketOwner)).Methods(http.MethodPut) diff --git a/weed/admin/handlers/admin_handlers_routes_test.go b/weed/admin/handlers/admin_handlers_routes_test.go index a34102d7e..521a4d318 100644 --- a/weed/admin/handlers/admin_handlers_routes_test.go +++ b/weed/admin/handlers/admin_handlers_routes_test.go @@ -42,6 +42,26 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) { } } +func TestSetupRoutes_RegistersBucketLifecycleAPI_NoAuth(t *testing.T) { + router := mux.NewRouter() + + newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true) + + assertHasRoute(t, router, http.MethodGet, "/api/s3/buckets/example/lifecycle") + assertHasRoute(t, router, http.MethodPut, "/api/s3/buckets/example/lifecycle") + assertHasRoute(t, router, http.MethodDelete, "/api/s3/buckets/example/lifecycle") +} + +func TestSetupRoutes_RegistersBucketLifecycleAPI_WithAuth(t *testing.T) { + router := mux.NewRouter() + + newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true) + + assertHasRoute(t, router, http.MethodGet, "/api/s3/buckets/example/lifecycle") + assertHasRoute(t, router, http.MethodPut, "/api/s3/buckets/example/lifecycle") + assertHasRoute(t, router, http.MethodDelete, "/api/s3/buckets/example/lifecycle") +} + func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) { router := mux.NewRouter() diff --git a/weed/admin/view/app/s3_buckets.templ b/weed/admin/view/app/s3_buckets.templ index 224de2adc..47714f1bd 100644 --- a/weed/admin/view/app/s3_buckets.templ +++ b/weed/admin/view/app/s3_buckets.templ @@ -637,7 +637,7 @@ templ S3Buckets(data dash.S3BucketsData) {