package s3api import ( "encoding/json" "github.com/aws/aws-sdk-go/service/s3" lru "github.com/hashicorp/golang-lru/v2" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "golang.org/x/sync/singleflight" ) var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) { entry, err := r.s3a.getBucketEntry(bucketName) if err != nil { return nil, err } return buildBucketMetadata(r.s3a.iam, entry), nil } type BucketMetaData struct { _ struct{} `type:"structure"` Name string // Indicates the bucket is a table bucket. IsTableBucket bool //By default, when another AWS account uploads an object to S3 bucket, //that account (the object writer) owns the object, has access to it, and //can grant other users access to it through ACLs. You can use Object Ownership //to change this default behavior so that ACLs are disabled and you, as the //bucket owner, automatically own every object in your bucket. ObjectOwnership string // Container for the bucket owner's display name and ID. Owner *s3.Owner `type:"structure"` // A list of grants for access controls. Acl []*s3.Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` } type BucketRegistry struct { metadataCache *lru.Cache[string, *BucketMetaData] notFound *lru.Cache[string, struct{}] // loadGroup deduplicates concurrent filer loads of the same bucket // without serializing loads of different buckets loadGroup singleflight.Group s3a *S3ApiServer } // NewBucketRegistry creates a lazy registry: nothing is listed at startup, // buckets load from the filer on first access and stay fresh via the // metadata subscription. func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry { metadataCache, _ := lru.New[string, *BucketMetaData](bucketCacheCapacity) notFound, _ := lru.New[string, struct{}](bucketCacheCapacity) return &BucketRegistry{ metadataCache: metadataCache, notFound: notFound, s3a: s3a, } } // LoadBucketMetadata refreshes a bucket already resident in the cache from a // subscription event. Cold buckets are left to lazy-load on first access so // the cache holds only this gateway's working set. func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) { if r.metadataCache.Contains(entry.Name) { r.metadataCache.Add(entry.Name, buildBucketMetadata(r.s3a.iam, entry)) } // Remove from notFound cache since bucket now exists r.unMarkNotFound(entry.Name) } // bucketOwnerAccountId returns the account id owning the bucket entry. A bucket // created outside the S3 API (the admin UI, weed shell) records only its owning // identity, so that identity is resolved to its account. Without this such a // bucket looks unowned: it reports the default admin account as its ACL owner, // and under BucketOwnerEnforced every object written to it is stamped with that // account instead of the bucket owner. func bucketOwnerAccountId(accountManager AccountManager, entry *filer_pb.Entry) string { if ownerAccountId := string(entry.Extended[s3_constants.ExtAmzOwnerKey]); ownerAccountId != "" { return ownerAccountId } if identityName := string(entry.Extended[s3_constants.AmzIdentityId]); identityName != "" { return accountManager.GetAccountIdByIdentityName(identityName) } return "" } func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *BucketMetaData { entryJson, _ := json.Marshal(entry) glog.V(3).Infof("build bucket metadata,entry=%s", entryJson) bucketMetadata := &BucketMetaData{ Name: entry.Name, IsTableBucket: s3tables.IsTableBucketEntry(entry), //Default ownership: OwnershipBucketOwnerEnforced, which means Acl is disabled ObjectOwnership: s3_constants.DefaultOwnershipForExists, // Default owner: `AccountAdmin` Owner: &s3.Owner{ ID: &AccountAdmin.Id, DisplayName: &AccountAdmin.DisplayName, }, } if entry.Extended != nil { //ownership control if ownership, ok := entry.Extended[s3_constants.ExtOwnershipKey]; ok { if !s3_constants.ValidateOwnership(string(ownership)) { glog.Warningf("Invalid ownership: %s, bucket: %s", string(ownership), bucketMetadata.Name) } bucketMetadata.ObjectOwnership = s3_constants.EffectiveOwnership(string(ownership)) } //access control policy //owner ownerAccountId := bucketOwnerAccountId(accountManager, entry) if ownerAccountId != "" { ownerAccountName := accountManager.GetAccountNameById(ownerAccountId) if ownerAccountName == "" { glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name) } else { bucketMetadata.Owner = &s3.Owner{ ID: &ownerAccountId, DisplayName: &ownerAccountName, } } } //grants acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey] if ok && len(acpGrantsBytes) > 0 { var grants []*s3.Grant err := json.Unmarshal(acpGrantsBytes, &grants) if err == nil { bucketMetadata.Acl = grants } else { glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name) } } } return bucketMetadata } func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) { r.removeMetadataCache(entry.Name) r.unMarkNotFound(entry.Name) } func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) { bucketMetadata, ok := r.metadataCache.Get(bucketName) if ok { return bucketMetadata, s3err.ErrNone } if r.notFound.Contains(bucketName) { return nil, s3err.ErrNoSuchBucket } return r.LoadBucketMetadataFromFiler(bucketName) } // LoadBucketMetadataFromFiler loads the bucket from the filer; concurrent // calls for the same bucket share one load, and the cache is filled inside // the flight so a bucket is fetched only once. func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*BucketMetaData, s3err.ErrorCode) { metadata, err, _ := r.loadGroup.Do(bucketName, func() (interface{}, error) { //check if already exists if bucketMetaData, ok := r.metadataCache.Get(bucketName); ok { return bucketMetaData, nil } //if not exists, load from filer bucketMetadata, err := loadBucketMetadataFromFiler(r, bucketName) if err != nil { if err == filer_pb.ErrNotFound { // The bucket doesn't actually exist and should no longer loaded from the filer r.notFound.Add(bucketName, struct{}{}) } return nil, err } r.setMetadataCache(bucketMetadata) r.unMarkNotFound(bucketName) return bucketMetadata, nil }) if err != nil { if err == filer_pb.ErrNotFound { return nil, s3err.ErrNoSuchBucket } return nil, s3err.ErrInternalError } return metadata.(*BucketMetaData), s3err.ErrNone } func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) { r.metadataCache.Add(metadata.Name, metadata) } func (r *BucketRegistry) removeMetadataCache(bucket string) { r.metadataCache.Remove(bucket) } func (r *BucketRegistry) unMarkNotFound(bucket string) { r.notFound.Remove(bucket) }