From 292c7493facc90218ab20753373caa98a6e6c5d4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 3 Jul 2026 12:39:45 -0700 Subject: [PATCH] s3: enforce bucket quota on logical size and surface read-only state in Admin UI (#10224) * s3: enforce bucket quota on logical size, not un-vacuumed physical size A bucket full of deleted/overwritten objects awaiting vacuum went read-only while its live data stayed under quota, because enforcement used the raw single-copy volume size with garbage included. Subtract DeletedByteCount via a LogicalSize() helper in the auto-enforce loop, the s3.bucket.quota.enforce command, and the bucket_size_bytes metric (labeled logical but counting garbage too). Deleting objects now relieves quota immediately and enforcement matches the UI usage figure. * admin: surface bucket read-only state in the S3 buckets UI Read the read-only flag quota enforcement writes to filer.conf and show it as a badge in the bucket list and a Status row in the details modal, so an operator can see why writes are being rejected. --- weed/admin/dash/admin_server.go | 37 ++++ weed/admin/dash/types.go | 1 + weed/admin/view/app/s3_buckets.templ | 11 ++ weed/admin/view/app/s3_buckets_templ.go | 182 +++++++++++--------- weed/s3api/bucket_size_metrics.go | 19 +- weed/s3api/bucket_size_metrics_test.go | 31 ++++ weed/shell/command_collection_list.go | 10 ++ weed/shell/command_s3_bucket_quota_check.go | 2 +- 8 files changed, 202 insertions(+), 91 deletions(-) diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 282d30647c..1bf10fec22 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -16,6 +16,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/cluster" clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance" "github.com/seaweedfs/seaweedfs/weed/credential" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -73,6 +74,27 @@ func (s *AdminServer) getFilerConfig() (*FilerConfig, error) { return config, err } +// getFilerConf reads filer.conf so callers can inspect per-path rules such as +// the read-only flag that quota enforcement toggles. A missing filer.conf +// yields an empty (all-writable) config rather than an error. +func (s *AdminServer) getFilerConf() (*filer.FilerConf, error) { + fc := filer.NewFilerConf() + err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + content, err := filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName) + if err != nil { + if errors.Is(err, filer_pb.ErrNotFound) { + return nil + } + return err + } + if len(content) > 0 { + return fc.LoadFromBytes(content) + } + return nil + }) + return fc, err +} + // getCollectionName returns the collection name for a bucket, prefixed with filer group if configured func getCollectionName(filerGroup, bucketName string) string { if filerGroup != "" { @@ -670,6 +692,12 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { glog.Warningf("Failed to get filer configuration, using defaults: %v", err) } + // Read filer.conf so we can surface the read-only flag quota enforcement sets + fc, err := s.getFilerConf() + if err != nil { + glog.Warningf("Failed to read filer.conf for bucket read-only state: %v", err) + } + // Now list buckets from the filer and match with collection data err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Paginate through all buckets in the buckets directory @@ -768,6 +796,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0) lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0) } + readOnly := fc.MatchStorageRule(filerConfig.BucketsPath + "/" + bucketName + "/").ReadOnly bucket := S3Bucket{ Name: bucketName, CreatedAt: createdAt, @@ -776,6 +805,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { LastModified: lastModified, Quota: quota, QuotaEnabled: quotaEnabled, + ReadOnly: readOnly, VersioningStatus: versioningStatus, ObjectLockEnabled: objectLockEnabled, ObjectLockMode: objectLockMode, @@ -829,6 +859,13 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error details.Bucket.PhysicalSize = data.PhysicalSize } + // Surface the read-only flag quota enforcement sets in filer.conf + if fc, err := s.getFilerConf(); err != nil { + glog.Warningf("Failed to read filer.conf for bucket read-only state: %v", err) + } else { + details.Bucket.ReadOnly = fc.MatchStorageRule(filerConfig.BucketsPath + "/" + bucketName + "/").ReadOnly + } + err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Get bucket info bucketResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ diff --git a/weed/admin/dash/types.go b/weed/admin/dash/types.go index 98d4d86f23..e8fcacb53c 100644 --- a/weed/admin/dash/types.go +++ b/weed/admin/dash/types.go @@ -79,6 +79,7 @@ type S3Bucket struct { LastModified time.Time `json:"last_modified"` Quota int64 `json:"quota"` // Quota in bytes, 0 means no quota QuotaEnabled bool `json:"quota_enabled"` // Whether quota is enabled + ReadOnly bool `json:"read_only"` // Whether the bucket path is read-only in filer.conf (e.g. quota enforcement) VersioningStatus string `json:"versioning_status"` // Versioning status: "" (never enabled), "Enabled", or "Suspended" ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE" diff --git a/weed/admin/view/app/s3_buckets.templ b/weed/admin/view/app/s3_buckets.templ index e268f3be63..218f287876 100644 --- a/weed/admin/view/app/s3_buckets.templ +++ b/weed/admin/view/app/s3_buckets.templ @@ -162,6 +162,11 @@ templ S3Buckets(data dash.S3BucketsData) { {bucket.Name} + if bucket.ReadOnly { + + Read-only + + } if bucket.Owner != "" { @@ -1087,6 +1092,11 @@ function displayBucketDetails(data) { versioningHtml = 'Suspended'; } + let statusHtml = 'Writable'; + if (bucket.read_only) { + statusHtml = 'Read-only'; + } + let objectLockHtml = 'Not configured'; if (bucket.object_lock_enabled) { let details = ''; @@ -1112,6 +1122,7 @@ function displayBucketDetails(data) { '
', '
Configuration
', '', + '") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "\" title=\"Delete Bucket\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Buckets) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
Status:' + statusHtml + '<\/td><\/tr>', '
Quota:' + quotaHtml + '<\/td><\/tr>', '
Versioning:' + versioningHtml + '<\/td><\/tr>', '
Object Lock:' + objectLockHtml + '<\/td><\/tr>', diff --git a/weed/admin/view/app/s3_buckets_templ.go b/weed/admin/view/app/s3_buckets_templ.go index 08b94e859e..7fc915c104 100644 --- a/weed/admin/view/app/s3_buckets_templ.go +++ b/weed/admin/view/app/s3_buckets_templ.go @@ -184,102 +184,112 @@ func S3Buckets(data dash.S3BucketsData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if bucket.ReadOnly { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "Read-only") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.Owner != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Owner) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 169, Col: 101} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 174, Col: 101} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "No owner") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "No owner") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 175, Col: 92} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 180, Col: 92} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var9 string templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(bucket.LogicalSize)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 177, Col: 85} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 182, Col: 85} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.PhysicalSize > 0 && bucket.LogicalSize > 0 && bucket.PhysicalSize > bucket.LogicalSize { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var10 string templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1fx overhead", float64(bucket.PhysicalSize)/float64(bucket.LogicalSize))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 180, Col: 144} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 185, Col: 144} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "		") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var11 string templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(bucket.PhysicalSize)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 184, Col: 81} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 189, Col: 81} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.Quota > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -288,7 +298,7 @@ func S3Buckets(data dash.S3BucketsData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var14 string templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(bucket.Quota)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 189, Col: 86} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 194, Col: 86} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.QuotaEnabled { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var15 string templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f%% used", float64(bucket.LogicalSize)/float64(bucket.Quota)*100)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 193, Col: 146} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 198, Col: 146} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "
Disabled
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "
Disabled
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "No quota") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "No quota") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "		") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.VersioningStatus == "Enabled" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "Enabled") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Enabled") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else if bucket.VersioningStatus == "Suspended" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "Suspended") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "Suspended") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Not configured") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "Not configured") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.ObjectLockEnabled { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
Enabled
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
Enabled
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var16 string templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ObjectLockMode) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 223, Col: 82} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 228, Col: 82} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, " • ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, " • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var17 string templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d days", bucket.ObjectLockDuration)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 223, Col: 138} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 228, Col: 138} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "Not configured") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "Not configured") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
No Object Store buckets found

Create your first bucket to get started with S3 storage.

No Object Store buckets found

Create your first bucket to get started with S3 storage.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.TotalPages > 1 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
Showing ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
Showing ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var26 string templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", (data.CurrentPage-1)*data.PageSize+1)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 292, Col: 101} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 297, Col: 101} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " to ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, " to ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var27 string templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", minInt(data.CurrentPage*data.PageSize, data.TotalBuckets))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 292, Col: 185} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 297, Col: 185} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, " of ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " of ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var28 string templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 292, Col: 229} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 297, Col: 229} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, " buckets
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "
Last updated: ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "
Last updated: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var34 string templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 352, Col: 81} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 357, Col: 81} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
Create New S3 Bucket
Bucket names must be between 3 and 63 characters, contain only lowercase letters, numbers, dots, and hyphens.
The S3 identity that owns this bucket. Non-admin users can only access buckets they own.
Set the maximum storage size for this bucket.
Keep multiple versions of objects in this bucket.
Prevent objects from being deleted or overwritten for a specified period. Automatically enables versioning.
Governance allows override with special permissions, Compliance is immutable.
Apply default retention to all new objects in this bucket.
Default retention period for new objects (1-36500 days).
Delete Bucket

Are you sure you want to delete the bucket ?

Warning: This action cannot be undone. All objects in the bucket will be permanently deleted.
Manage Bucket Quota
Set the maximum storage size for this bucket. Set to 0 to remove quota.
Bucket Details
Loading bucket details...
Manage Bucket Owner
Select the S3 identity that owns this bucket. Non-admin users can only access buckets they own.
Loading users...
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, "
Create New S3 Bucket
Bucket names must be between 3 and 63 characters, contain only lowercase letters, numbers, dots, and hyphens.
The S3 identity that owns this bucket. Non-admin users can only access buckets they own.
Set the maximum storage size for this bucket.
Keep multiple versions of objects in this bucket.
Prevent objects from being deleted or overwritten for a specified period. Automatically enables versioning.
Governance allows override with special permissions, Compliance is immutable.
Apply default retention to all new objects in this bucket.
Default retention period for new objects (1-36500 days).
Delete Bucket

Are you sure you want to delete the bucket ?

Warning: This action cannot be undone. All objects in the bucket will be permanently deleted.
Manage Bucket Quota
Set the maximum storage size for this bucket. Set to 0 to remove quota.
Bucket Details
Loading bucket details...
Manage Bucket Owner
Select the S3 identity that owns this bucket. Non-admin users can only access buckets they own.
Loading users...
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/s3api/bucket_size_metrics.go b/weed/s3api/bucket_size_metrics.go index 7ab29c2375..18cdffa98d 100644 --- a/weed/s3api/bucket_size_metrics.go +++ b/weed/s3api/bucket_size_metrics.go @@ -30,11 +30,22 @@ type CollectionInfo struct { FileCount float64 DeleteCount float64 DeletedByteCount float64 - Size float64 // Logical size (deduplicated by volume ID) + Size float64 // Single-copy volume size (deduplicated by volume ID), still counting un-vacuumed garbage PhysicalSize float64 // Physical size (including all replicas) VolumeCount int // Logical volume count (deduplicated by volume ID) } +// LogicalSize is the live data size: single-copy volume size minus the +// un-vacuumed deleted/overwritten bytes. Quota enforcement and the +// bucket_size_bytes metric use this so vacuum lag never counts against a +// bucket, matching the usage figure the Admin UI shows. +func (c *CollectionInfo) LogicalSize() float64 { + if c.Size < c.DeletedByteCount { + return 0 + } + return c.Size - c.DeletedByteCount +} + // volumeKey uniquely identifies a volume for deduplication type volumeKey struct { collection string @@ -105,9 +116,9 @@ func (s3a *S3ApiServer) collectAndUpdateBucketSizeMetrics(ctx context.Context) { for _, bucket := range buckets { collection := s3a.getCollectionName(bucket.Name) if info, found := collectionInfos[collection]; found { - stats.UpdateBucketSizeMetrics(bucket.Name, info.Size, info.PhysicalSize, info.FileCount) + stats.UpdateBucketSizeMetrics(bucket.Name, info.LogicalSize(), info.PhysicalSize, info.FileCount) glog.V(3).Infof("Updated bucket size metrics: bucket=%s, logicalSize=%.0f, physicalSize=%.0f, objects=%.0f", - bucket.Name, info.Size, info.PhysicalSize, info.FileCount) + bucket.Name, info.LogicalSize(), info.PhysicalSize, info.FileCount) } else { // Bucket exists but no collection data (empty bucket) stats.UpdateBucketSizeMetrics(bucket.Name, 0, 0, 0) @@ -134,7 +145,7 @@ func (s3a *S3ApiServer) enforceBucketQuotas(ctx context.Context, buckets []*file for _, bucket := range buckets { var size float64 if info, found := collectionInfos[s3a.getCollectionName(bucket.Name)]; found { - size = info.Size + size = info.LogicalSize() } locPrefix := s3a.option.BucketsPath + "/" + bucket.Name + "/" readOnly, flipped := fc.ApplyBucketQuotaReadOnly(locPrefix, size, float64(bucket.Quota)) diff --git a/weed/s3api/bucket_size_metrics_test.go b/weed/s3api/bucket_size_metrics_test.go index 259d0b0985..08c21e79f6 100644 --- a/weed/s3api/bucket_size_metrics_test.go +++ b/weed/s3api/bucket_size_metrics_test.go @@ -151,6 +151,11 @@ func TestCollectCollectionInfoFromTopologyMixed(t *testing.T) { if info.Size != 5000+6000 { t.Errorf("Size: got %.0f, want 11000", info.Size) } + // LogicalSize drops the 100 bytes of un-vacuumed garbage on the regular + // volume; EC shards carry no DeletedByteCount here. + if info.LogicalSize() != 11000-100 { + t.Errorf("LogicalSize: got %.0f, want 10900", info.LogicalSize()) + } if info.FileCount != 50+80 { t.Errorf("FileCount: got %.0f, want 130", info.FileCount) } @@ -209,3 +214,29 @@ func TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe(t *testing.T) { t.Errorf("FileCount: got %.0f, want 6 (max across reporters)", info.FileCount) } } + +// TestCollectionInfoLogicalSize verifies logical size excludes un-vacuumed +// garbage and never goes negative. Quota enforcement runs on this value so a +// bucket full of tombstones is not flipped read-only while its live data is +// under quota. +func TestCollectionInfoLogicalSize(t *testing.T) { + cases := []struct { + name string + size float64 + deleted float64 + expected float64 + }{ + {"no garbage", 1000, 0, 1000}, + {"some garbage", 1000, 300, 700}, + {"all garbage", 1000, 1000, 0}, + {"deleted exceeds size clamps to zero", 300, 1000, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + info := &CollectionInfo{Size: tc.size, DeletedByteCount: tc.deleted} + if got := info.LogicalSize(); got != tc.expected { + t.Errorf("LogicalSize(): got %.0f, want %.0f", got, tc.expected) + } + }) + } +} diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go index 916b844317..0287b1e96c 100644 --- a/weed/shell/command_collection_list.go +++ b/weed/shell/command_collection_list.go @@ -37,6 +37,16 @@ type CollectionInfo struct { VolumeCount int } +// LogicalSize is the live data size: single-copy volume size minus the +// un-vacuumed deleted/overwritten bytes. Quota enforcement uses this so +// vacuum lag never counts against a bucket. +func (c *CollectionInfo) LogicalSize() float64 { + if c.Size < c.DeletedByteCount { + return 0 + } + return c.Size - c.DeletedByteCount +} + func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { collections, err := ListCollectionNames(commandEnv, true, true) diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go index c72f043d51..7313514ec2 100644 --- a/weed/shell/command_s3_bucket_quota_check.go +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -74,7 +74,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, collection := getCollectionName(commandEnv, entry.Name) var collectionSize float64 if collectionInfo, found := collectionInfos[collection]; found { - collectionSize = collectionInfo.Size + collectionSize = collectionInfo.LogicalSize() } if c.processEachBucket(fc, filerBucketsPath, entry, writer, collectionSize) { hasConfChanges = true