From d6ec6bc4e7df067c1b2175905895097aef70372f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20L=E1=BB=99c=20Ph=C3=BAc?= <112325531+omninguyen@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:43:15 +0700 Subject: [PATCH 01/31] fix(s3tables): let DeleteTableBucket remove empty leftover directories (#12307) * s3tables: let DeleteTableBucket remove empty leftover directories Page through the bucket instead of reading one page, and check each bare directory a deleted namespace left behind for files: empty ones go with the bucket, one holding files refuses with 409 naming it. The delete claims the bucket before scanning, and CreateNamespace refuses a claimed bucket and confirms it again after its write, so a namespace created meanwhile is seen by the scan or turned away instead of deleted with the bucket. The bucket is removed only while it still carries that claim, otherwise 409. A delete that refuses lifts the claim, retrying a failed release, since a claim left behind refuses every create in the bucket. The new namespace stays marked unfinished until that check passes, so no table or view is created in one that gets backed out; a create takes over a namespace still carrying the mark. Both checks accept only the bucket the create was authorized against, by its creation time: one recreated under the same name answers 404 and gets nothing written into it. The removal is not recursive. Each leftover is claimed first, only while it is still the directory the scan saw, and goes only while it holds no file or catalog record; the bucket goes only while nothing is left in it. A file uploaded or a namespace taken back after the scan is refused with 409 instead of deleted. * s3tables: keep directory-backed S3 objects in DeleteTableBucket * s3tables: keep a file that replaced a leftover directory in DeleteTableBucket * s3tables: claim only the authorized bucket in DeleteTableBucket * s3tables: assert the prefix-object mark when claiming a leftover directory * s3tables: hide unfinished namespaces from readers and DeleteNamespace * s3tables: stop CreateNamespace on any failed bucket check * s3tables: retry lifting claims on leftover directories * s3vectors: declare the index namespace again after losing it to a concurrent declare CreateNamespace now writes a namespace marked unfinished and confirms it after checking the bucket. A second CreateIndex that declares the namespace in that window takes the marked record over, and the first one's confirmation answers 409, which failed its CreateIndex. Declare it again, a bounded number of times, when the create answers a conflict. Co-Authored-By: Claude Opus 5.5 * s3tables: name an unfinished namespace that keeps a bucket from being deleted A CreateNamespace that dies between its write and its confirmation leaves a namespace marked unfinished. Listings and GetNamespace hide it and DeleteNamespace answers that it does not exist, while DeleteTableBucket counts it, so the bucket could not be deleted and nothing said why. Name it in the BucketNotEmpty answer; creating it again takes it over. Co-Authored-By: Claude Opus 5.5 * s3tables: read a refused empty-directory delete with filer.DeleteEntryError Use the filer's own reading of a non-recursive delete's answer instead of a copy of its message prefix. Co-Authored-By: Claude Opus 5.5 * s3vectors: release the held create when the concurrent-declare test fails Co-Authored-By: Claude Opus 5.5 * s3vectors: buffer the held create's result so it can exit after a failure Co-Authored-By: Claude Opus 5.5 * s3tables: retry lifting a refused namespace delete's claim A claim left on a namespace refuses every create in it, the same harm the bucket's claim does, so its release retries too. With no one-shot caller left, the retrying release takes the releaseDeleteClaim name. Co-Authored-By: Claude Opus 5.5 * s3tables: keep the data under a hidden directory on bucket delete The scan filed every hidden entry by name for a recursive delete, so .hidden/data went with the bucket. A hidden directory waits with the leftovers instead: the walk refuses one that holds a file or an object, and removeEmptyTree takes only one that is still bare and empty. A hidden file is removed by name as before. --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- weed/s3api/s3tables/filer_ops.go | 300 +++++- .../handler_bucket_get_list_delete.go | 157 ++- .../handler_bucket_keeps_data_test.go | 997 ++++++++++++++++++ .../handler_delete_create_race_test.go | 16 + weed/s3api/s3tables/handler_namespace.go | 96 +- weed/s3api/s3tables/handler_table.go | 12 +- weed/s3api/s3tables/s3tablestest/memfiler.go | 12 +- weed/s3api/s3vectors/handlers.go | 52 +- weed/s3api/s3vectors/handlers_test.go | 53 + 9 files changed, 1628 insertions(+), 67 deletions(-) create mode 100644 weed/s3api/s3tables/handler_bucket_keeps_data_test.go diff --git a/weed/s3api/s3tables/filer_ops.go b/weed/s3api/s3tables/filer_ops.go index 96fae0a259..774743dd69 100644 --- a/weed/s3api/s3tables/filer_ops.go +++ b/weed/s3api/s3tables/filer_ops.go @@ -11,8 +11,10 @@ import ( "path" "time" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -107,6 +109,20 @@ func (h *S3TablesHandler) mutateEntry( client filer_pb.SeaweedFilerClient, path string, mutate func(entry *filer_pb.Entry) error, +) error { + return h.mutateEntryAsserting(ctx, client, path, nil, mutate) +} + +// mutateEntryAsserting is mutateEntry with the keys in absent asserted at the +// write too, as this package's own attributes are: one absent when the entry was +// read and written by someone else since fails the compare, instead of being +// written over with the entry as it was read. +func (h *S3TablesHandler) mutateEntryAsserting( + ctx context.Context, + client filer_pb.SeaweedFilerClient, + path string, + absent []string, + mutate func(entry *filer_pb.Entry) error, ) error { dir, name := splitPath(path) @@ -120,7 +136,7 @@ func (h *S3TablesHandler) mutateEntry( } entry := resp.Entry - expected := snapshotExtended(entry.Extended) + expected := snapshotExtended(entry.Extended, absent...) if entry.Extended == nil { entry.Extended = make(map[string][]byte) } @@ -401,13 +417,25 @@ func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer // readNamespaceMetadataAttr returns the namespace-metadata xattr at // namespacePath. A directory without it is not a namespace — a delete that kept -// the data under it leaves one — so its absence reads as a missing entry. +// the data under it leaves one — so its absence reads as a missing entry. So +// does a namespace still marked by its create: like a table's, it is absent to +// every reader until the create confirms it. func (h *S3TablesHandler) readNamespaceMetadataAttr(ctx context.Context, client filer_pb.SeaweedFilerClient, namespacePath string) ([]byte, error) { - data, err := h.getExtendedAttribute(ctx, client, namespacePath, ExtendedKeyMetadata) - if errors.Is(err, ErrAttributeNotFound) { + entry, err := h.lookupEntry(ctx, client, namespacePath) + if err != nil { + return nil, err + } + data, ok := entry.Extended[ExtendedKeyMetadata] + if !ok || isUnfinishedCreate(entry.Extended) { return nil, fmt.Errorf("%w: %s", filer_pb.ErrNotFound, namespacePath) } - return data, err + return data, nil +} + +// isFinishedNamespaceEntry is isNamespaceEntry for a namespace its create has +// confirmed, the only kind a reader or a delete may act on. +func isFinishedNamespaceEntry(entry *filer_pb.Entry) bool { + return isNamespaceEntry(entry) && !isUnfinishedCreate(entry.Extended) } // loadNamespaceMetadata resolves a namespace to its metadata. A directory that @@ -499,9 +527,46 @@ func (h *S3TablesHandler) deleteDirectoryIfMatches(ctx context.Context, client f return nil } +// errDirNotEmpty is a directory that holds something the caller meant to leave +// alone: a file in a leftover tree, or an entry the filer found in a directory +// it was asked to delete without its contents. +var errDirNotEmpty = errors.New("directory is not empty") + +// deleteEmptyDirectoryIfMatches deletes a directory without its contents, only +// while it still carries the extended attributes the caller captured. The +// filer checks both under the path lock, in the one delete: a changed +// attribute is ErrConcurrentUpdate, and an entry that appeared in the +// directory is errDirNotEmpty, so neither a directory taken back since the +// caller looked at it nor a file written into it since is lost. The mark of an +// empty S3 object stored on the directory is asserted absent too, so one +// written since is kept the same way. +func (h *S3TablesHandler) deleteEmptyDirectoryIfMatches(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, expected *filer_pb.Entry) error { + dir, name := splitPath(path) + resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{ + Directory: dir, + Name: name, + IsDeleteData: true, + ExpectedExtended: snapshotExtended(expected.Extended, s3_constants.SeaweedFSPrefixObject), + }) + if status.Code(err) == codes.FailedPrecondition { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path) + } + if err != nil { + return err + } + if resp != nil && resp.Error != "" { + if err := filer.DeleteEntryError(resp.Error); !errors.Is(err, filer.ErrNonEmptyFolder) { + return err + } + return fmt.Errorf("%w: %s", errDirNotEmpty, path) + } + return nil +} + var ( errParentDeleting = errors.New("parent is being deleted") errParentGone = errors.New("parent was deleted") + errParentCreating = errors.New("parent is still being created") errNoDeleteClaim = errors.New("claim not held") errBackOutFailed = errors.New("backing out the create failed") errSourceGone = errors.New("rename source was deleted") @@ -527,18 +592,54 @@ func (h *S3TablesHandler) claimForDelete(ctx context.Context, client filer_pb.Se return claim, claimed, nil } -// releaseDeleteClaim removes claim from path while path still holds it, even -// after the request was cancelled. -func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) { - err := h.mutateEntryExtended(context.WithoutCancel(ctx), client, path, func(extended map[string][]byte) error { +// tryReleaseDeleteClaim removes claim from path once, while path still holds +// it. A claim another delete replaced or lifted, or an entry that is gone, +// leaves nothing to release and is not an error. +func (h *S3TablesHandler) tryReleaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) error { + err := h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error { if string(extended[ExtendedKeyDeleting]) != claim { return errNoDeleteClaim } delete(extended, ExtendedKeyDeleting) return nil }) - if err != nil && !errors.Is(err, errNoDeleteClaim) && !errors.Is(err, filer_pb.ErrNotFound) { - glog.Warningf("release delete claim on %s: %v", path, err) + if errors.Is(err, errNoDeleteClaim) || errors.Is(err, filer_pb.ErrNotFound) { + return nil + } + return err +} + +// releaseDeleteClaim removes claim from path while path still holds it, even +// after the request was cancelled. A claim left behind refuses creates under a +// bucket or namespace and makes a leftover directory read as taken, so a failed +// release is retried, including one that lost to another write of the entry. +func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) { + ctx = context.WithoutCancel(ctx) + var err error + for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ { + if attempt > 0 { + time.Sleep(100 * time.Millisecond) + } + if err = h.tryReleaseDeleteClaim(ctx, client, path, claim); err == nil { + return + } + } + glog.Warningf("release delete claim on %s: %v", path, err) +} + +// isTableBucketCreatedAt accepts only the table bucket created at createdAt. A +// bucket deleted and created again under the same name has a CreatedAt of its +// own, so a create authorized against the first reads the second as gone. +func isTableBucketCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool { + return func(entry *filer_pb.Entry) bool { + if !IsTableBucketEntry(entry) { + return false + } + var metadata tableBucketMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil { + return false + } + return metadata.CreatedAt.Equal(createdAt) } } @@ -549,7 +650,8 @@ func isNamespaceEntry(entry *filer_pb.Entry) bool { } // checkParentOpen returns errParentGone if parentPath is missing or not a -// catalog entry, and errParentDeleting if a delete has claimed it. +// catalog entry, errParentCreating if the create that wrote it has not +// confirmed it yet, and errParentDeleting if a delete has claimed it. func (h *S3TablesHandler) checkParentOpen(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool) error { _, err := h.openParent(ctx, client, parentPath, isParent) return err @@ -568,6 +670,9 @@ func (h *S3TablesHandler) openParent(ctx context.Context, client filer_pb.Seawee if !isParent(entry) { return nil, errParentGone } + if isUnfinishedCreate(entry.Extended) { + return nil, errParentCreating + } if _, claimed := entry.Extended[ExtendedKeyDeleting]; claimed { return nil, errParentDeleting } @@ -644,6 +749,20 @@ func (h *S3TablesHandler) mutateCatalogRecord( // it removes what the create wrote and returns why, or errBackOutFailed if // the record stays. func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error { + return h.confirmChild(ctx, client, parentPath, isParent, true, childPath, written, emptyDirs) +} + +// confirmParentOnce is confirmParent for a record that other creates build on, +// a namespace. Checking the parent again after the mark comes off would leave a +// window in which a child is created in the record and the check then removes +// the record under it, so the parent is checked once, before the mark comes +// off. That is enough while a delete of the parent counts the marked record: one +// claiming the parent after the check scans the record and is refused. +func (h *S3TablesHandler) confirmParentOnce(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error { + return h.confirmChild(ctx, client, parentPath, isParent, false, childPath, written, emptyDirs) +} + +func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, recheck bool, childPath string, written map[string][]byte, emptyDirs []string) error { parent, err := h.openParent(ctx, client, parentPath, isParent) confirmed := false if err == nil { @@ -651,7 +770,9 @@ func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.Sea // create wrote; a failed write backs the record out like a closed parent. if err = h.removeExtendedAttributesIf(ctx, client, childPath, written, ExtendedKeyDeleting); err == nil { confirmed = true - err = h.parentStillOpen(ctx, client, parentPath, isParent, parent) + if recheck { + err = h.parentStillOpen(ctx, client, parentPath, isParent, parent) + } } } if err == nil { @@ -753,6 +874,159 @@ func (h *S3TablesHandler) dirHoldsFiles(ctx context.Context, client filer_pb.Sea return false, nil } +// claimTreeDirectory writes a fresh delete claim onto the directory at dirPath +// while it still carries expected, and returns the claim with the attributes it +// left. A claim seen in expected, one a delete that died left, is replaced like +// any other attribute; one written since fails the compare. +// +// The claim is what ties the later delete to this directory: the delete +// requires it, and only this directory carries it. A file or an S3 object at +// dirPath is refused with errDirNotEmpty, whether it was there when the entry +// was read or took the directory's place before the write, which the filer +// refuses since it never writes a directory entry over a file. One that takes +// the place after the write lacks the claim, so the delete is refused instead. +// An S3 object landing on the directory between the read and the write carries +// the prefix-object mark, which the write asserts absent, so the write fails +// instead of putting the bare directory back over the object. +func (h *S3TablesHandler) claimTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, expected map[string][]byte) (string, map[string][]byte, error) { + claim := generateVersionToken() + var claimed map[string][]byte + err := h.mutateEntryAsserting(ctx, client, dirPath, []string{s3_constants.SeaweedFSPrefixObject}, func(entry *filer_pb.Entry) error { + if !entry.IsDirectory || entry.IsDirectoryKeyObject() { + return fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath) + } + for key, want := range expected { + if !bytes.Equal(entry.Extended[key], want) { + return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, dirPath) + } + } + entry.Extended[ExtendedKeyDeleting] = []byte(claim) + claimed = snapshotExtended(entry.Extended) + return nil + }) + if errors.Is(err, filer_pb.ErrNotFound) { + return "", nil, fmt.Errorf("%w: %s", ErrConcurrentUpdate, dirPath) + } + if err != nil && !errors.Is(err, errDirNotEmpty) && !errors.Is(err, ErrConcurrentUpdate) { + // The filer reports the refused write over a file only as an opaque + // failure, so look at what holds the path now. + if current, lookupErr := h.lookupEntry(ctx, client, dirPath); lookupErr == nil && !current.IsDirectory { + return "", nil, fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath) + } + } + if err != nil { + return "", nil, err + } + return claim, claimed, nil +} + +// removeTreeDirectory removes a directory the walk listed as listed, under a +// claim of its own, and lifts the claim if the delete is refused. +func (h *S3TablesHandler) removeTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, listed *filer_pb.Entry) error { + claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(listed.Extended)) + if err != nil { + return err + } + if err := h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}); err != nil { + h.releaseDeleteClaim(ctx, client, dirPath, claim) + return err + } + return nil +} + +// removeEmptyTree removes dirPath and every directory under it, provided no +// file, no S3 object stored on a directory entry, and no catalog record sits +// anywhere in the tree, dirPath included. dirEntry is dirPath as the caller saw +// it. +// +// dirPath is claimed first, in a write that requires the attributes the caller +// saw: a directory taken back as a namespace since then fails it, before +// anything under it is touched, and the claim keeps it from being taken back +// while the tree goes. With dirPath no namespace, nothing under it can become a +// catalog record either, so a record the walk meets was there before and is +// refused, not removed. +// +// The walk lists the tree a page at a time and stops at the first file, object +// or record with errDirNotEmpty, before anything is removed. Directories then go +// deepest first, each claimed with the attributes it had when listed and +// deleted under that claim, which refuses a directory that holds an entry by +// now or was replaced by a file. dirPath goes last, still under its claim; any +// other way out lifts the claim. +func (h *S3TablesHandler) removeEmptyTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, dirEntry *filer_pb.Entry) (err error) { + claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(dirEntry.Extended)) + if err != nil { + return err + } + defer func() { + if err != nil { + h.releaseDeleteClaim(ctx, client, dirPath, claim) + } + }() + + type listedDir struct { + path string + entry *filer_pb.Entry + } + listed := []listedDir{{dirPath, &filer_pb.Entry{Extended: claimed}}} + + walkCtx, cancel := context.WithCancel(ctx) + defer cancel() + // A directory is listed after its parent, so removing in reverse order + // removes every directory after the ones under it. + for next := 0; next < len(listed); next++ { + dir := listed[next].path + lastFileName := "" + for { + resp, err := client.ListEntries(walkCtx, &filer_pb.ListEntriesRequest{ + Directory: dir, + Limit: dirListPageSize, + StartFromFileName: lastFileName, + InclusiveStartFrom: lastFileName == "", + }) + if err != nil { + if status.Code(err) == codes.NotFound { + break // gone, so its delete below reports the change + } + return err + } + + received := false + for { + entry, err := resp.Recv() + if err != nil { + if err == io.EOF || status.Code(err) == codes.NotFound { + break + } + return err + } + if entry.Entry == nil { + continue + } + received = true + lastFileName = entry.Entry.Name + child := path.Join(dir, entry.Entry.Name) + if !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() { + return fmt.Errorf("%w: %s is an object", errDirNotEmpty, child) + } + if _, isRecord := entry.Entry.Extended[ExtendedKeyMetadata]; isRecord { + return fmt.Errorf("%w: %s is a catalog record", errDirNotEmpty, child) + } + listed = append(listed, listedDir{child, entry.Entry}) + } + if !received { + break + } + } + } + + for i := len(listed) - 1; i > 0; i-- { + if err := h.removeTreeDirectory(ctx, client, listed[i].path, listed[i].entry); err != nil { + return err + } + } + return h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}) +} + // keepAsLeftover clears a namespace's catalog attributes and the delete's // claim in one write, while the entry still holds claimed, keeping the // directory and the data under it. The write compares against claimed so the diff --git a/weed/s3api/s3tables/handler_bucket_get_list_delete.go b/weed/s3api/s3tables/handler_bucket_get_list_delete.go index 1b801eb84a..2e7c604988 100644 --- a/weed/s3api/s3tables/handler_bucket_get_list_delete.go +++ b/weed/s3api/s3tables/handler_bucket_get_list_delete.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "path" "strings" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -275,6 +276,32 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http var metadata tableBucketMetadata var bucketPolicy string hasChildren := false + heldDir := "" + unfinishedNamespace := "" + // What the scan found besides namespaces: the bare directories — a deleted + // namespace's leftovers or hidden ones — as they were seen, and the names + // of the hidden files. + var leftovers []*filer_pb.Entry + var hidden []string + // The claim this delete holds on the bucket. It goes with the bucket when it + // is deleted; any other way out lifts it, or the bucket would refuse every + // create. bucketClaimed is the bucket's record as the claim left it, which the + // removal has to find. + bucketClaim := "" + var bucketClaimed map[string][]byte + defer func() { + if bucketClaim == "" { + return + } + // A claim left behind refuses every create in the bucket until the next + // delete replaces it, so a failed release is retried. + if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + h.releaseDeleteClaim(r.Context(), client, bucketPath, bucketClaim) + return nil + }); err != nil { + glog.Warningf("release delete claim on %s: %v", bucketPath, err) + } + }() err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // 1. Get metadata for ownership check data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata) @@ -310,25 +337,89 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName)) } - // 3. Check if bucket is empty - resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{ - Directory: bucketPath, - Limit: 10, - }) + // 2. Claim the bucket before scanning it. CreateNamespace refuses a + // claimed bucket and checks it again after its write, so a namespace is + // either seen by the scan below or backed out. The bucket is removed only + // while it still carries this claim: once another delete has replaced or + // lifted it, a create may have passed that check, and this delete answers + // 409 instead. A claim an earlier delete left behind is replaced. Only the + // bucket authorized above is claimed: one deleted and created again under + // the same name since then is another bucket and reads as gone. + bucketClaim, bucketClaimed, err = h.claimForDelete(r.Context(), client, bucketPath, isTableBucketCreatedAt(metadata.CreatedAt)) if err != nil { return err } - for { - entry, err := resp.Recv() + // 3. Check if bucket is empty. A file, an S3 object stored on a directory + // entry, or a namespace is a child. A bare directory is what a deleted + // namespace leaves behind to keep the data under it, so it is only a + // child while it still holds a file. The scan + // pages through the whole bucket: the first page may be nothing but bare + // directories sorted ahead of the namespaces. + lastFileName := "" + for !hasChildren { + resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{ + Directory: bucketPath, + Limit: 10, + StartFromFileName: lastFileName, + InclusiveStartFrom: lastFileName == "", + }) if err != nil { - if err == io.EOF { - break - } return err } - if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") { - hasChildren = true + + hasMore := false + for { + entry, err := resp.Recv() + if err != nil { + if err == io.EOF { + break + } + return err + } + if entry.Entry == nil { + continue + } + hasMore = true + lastFileName = entry.Entry.Name + // Hidden entries are never children. A hidden directory can + // hold files the same way a leftover does, so it waits with + // them for the walk; a hidden file is removed by name. + if strings.HasPrefix(entry.Entry.Name, ".") { + if entry.Entry.IsDirectory { + leftovers = append(leftovers, entry.Entry) + } else { + hidden = append(hidden, entry.Entry.Name) + } + continue + } + if _, hasMetadata := entry.Entry.Extended[ExtendedKeyMetadata]; hasMetadata || !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() { + hasChildren = true + if hasMetadata && isUnfinishedCreate(entry.Entry.Extended) { + unfinishedNamespace = entry.Entry.Name + } + break + } + leftovers = append(leftovers, entry.Entry) + } + if !hasMore { + break + } + } + + // Looked at after the scan, not during it, so a bucket that still has a + // namespace answers at once instead of walking the leftovers first. The + // first directory holding a file settles it. + if hasChildren { + return nil + } + for _, leftover := range leftovers { + holdsFiles, err := h.dirHoldsFiles(r.Context(), client, path.Join(bucketPath, leftover.Name)) + if err != nil { + return err + } + if holdsFiles { + heldDir = leftover.Name break } } @@ -341,6 +432,8 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) } else if isAuthError(err) { h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + } else if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket changed during delete, retry the request") } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to delete table bucket: %v", err)) } @@ -348,25 +441,59 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http } if hasChildren { - h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty") + message := "table bucket is not empty" + if unfinishedNamespace != "" { + // Listings hide it, so name it: creating it again finishes it. + message = fmt.Sprintf("table bucket is not empty: namespace %s has an unfinished create", unfinishedNamespace) + } + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, message) return fmt.Errorf("bucket not empty") } + // Nothing in the bucket is a namespace, but data sits under a directory a + // deleted namespace left. Deleting the bucket would take it, so it stays. + if heldDir != "" { + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, fmt.Sprintf("table bucket is not empty: directory %s still holds files", heldDir)) + return fmt.Errorf("bucket not empty: directory %s still holds files", heldDir) + } + // Delete the bucket err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Drop the leaf entry first, so it does not outlive the directory. if tableObjErr := h.deleteEntryIfExists(r.Context(), client, GetTableObjectBucketPath(bucketName)); tableObjErr != nil { glog.V(1).Infof("failed to delete table object for %s: %v", bucketName, tableObjErr) } + // Only the hidden files go by a recursive delete. A file written since + // the scan anywhere else makes the filer refuse the delete that would have + // taken it: each leftover goes only if it is still the bare directory the + // scan saw and holds no file or catalog record, and the bucket only if + // nothing is left in it. + for _, leftover := range leftovers { + if err := h.removeEmptyTree(r.Context(), client, path.Join(bucketPath, leftover.Name), leftover); err != nil { + return err + } + } + for _, name := range hidden { + if err := h.deleteDirectory(r.Context(), client, path.Join(bucketPath, name)); err != nil { + return err + } + } // The bucket is the directory, so a refused delete leaves it in place // and the caller must hear about it. - return h.deleteDirectory(r.Context(), client, bucketPath) + return h.deleteEmptyDirectoryIfMatches(r.Context(), client, bucketPath, &filer_pb.Entry{Extended: bucketClaimed}) }) if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket") + if errors.Is(err, errDirNotEmpty) { + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty") + } else if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket changed during delete, retry the request") + } else { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket") + } return err } + bucketClaim = "" h.writeJSON(w, http.StatusOK, nil) return nil diff --git a/weed/s3api/s3tables/handler_bucket_keeps_data_test.go b/weed/s3api/s3tables/handler_bucket_keeps_data_test.go new file mode 100644 index 0000000000..dfeef51285 --- /dev/null +++ b/weed/s3api/s3tables/handler_bucket_keeps_data_test.go @@ -0,0 +1,997 @@ +package s3tables + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// startEmptyBucket seeds a table bucket with nothing in it, not even a namespace. +func startEmptyBucket(t *testing.T) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs := s3tablestest.Start(t) + bucketMeta, err := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID}) + require.NoError(t, err) + fs.Put(TablesPath, renameTestBucket, map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: bucketMeta, + }) + m := NewManager() + m.SetTrusted(true) + return fs, m +} + +func deleteTableBucketRequest(t *testing.T) *DeleteTableBucketRequest { + return &DeleteTableBucketRequest{TableBucketARN: mustBucketARN(t)} +} + +// Deleting a namespace that holds renamed-away data leaves its directory behind +// with no record. Once the renamed table is dropped the data is gone, and the +// directory is an empty leftover that must not keep the bucket from being deleted. +func TestDeleteTableBucketRemovesEmptyLeftovers(t *testing.T) { + fs, m := startDeletedNamespaceWithRenamedData(t) + bucketPath := GetTableBucketPath(renameTestBucket) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the delete of ns keeps its directory, which is the leftover under test") + + requireOK(t, m, fs, "DeleteTable", &DeleteTableRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + Name: "t", + }) + requireOK(t, m, fs, "DeleteNamespace", &DeleteNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + require.Nil(t, fs.Get(keptDataDir(), keptDataFile), "dropping the renamed table purges the data the leftover kept") + require.NotNil(t, fs.Get(bucketPath, "ns"), "the leftover directory is still in the bucket") + + requireOK(t, m, fs, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + assert.Nil(t, fs.Get(bucketPath, "ns"), "the empty leftover goes with it") +} + +// No table refers to a file under a directory with no record, but deleting the +// bucket would destroy it without anyone deciding to. The bucket stays, and so +// does the file. +func TestDeleteTableBucketRefusesOrphanData(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + orphan := path.Join(bucketPath, "old", "t", "data") + fs.Put(bucketPath, "old", nil) + fs.Put(path.Join(bucketPath, "old"), "t", nil) + fs.Put(path.Join(bucketPath, "old", "t"), "data", nil) + fs.PutFile(orphan, keptDataFile, time.Now()) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.Contains(t, resp.Message, "directory old still holds files", "the answer names the directory that holds the file") + assert.NotNil(t, fs.Get(orphan, keptDataFile), "the file must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A hidden directory can hold data the same way a visible leftover does. +// Files under it refuse the delete; only a tree holding none goes with the +// bucket, and so does a hidden file. +func TestDeleteTableBucketKeepsDataUnderAHiddenDirectory(t *testing.T) { + cases := []struct { + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) + kept bool + }{ + {"files under a hidden directory", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.Put(bucketPath, ".hidden", nil) + fs.Put(hiddenPath, "data", nil) + fs.PutFile(path.Join(hiddenPath, "data"), keptDataFile, time.Now()) + }, true}, + {"only empty hidden directories", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.Put(bucketPath, ".hidden", nil) + fs.Put(hiddenPath, "data", nil) + }, false}, + {"a hidden file", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.PutFile(bucketPath, ".DS_Store", time.Now()) + }, false}, + {"a hidden directory stored as an object", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + putDirectoryObject(t, fs, bucketPath, ".hidden") + }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + tc.seed(t, fs, bucketPath, path.Join(bucketPath, ".hidden")) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + if tc.kept { + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(bucketPath, ".hidden"), "the hidden entry must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + } else { + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + } + }) + } +} + +// pauseListClient runs pause just before the nth listing of dir, so a test can +// land a competing write at a chosen step of a delete's directory walk. +type pauseListClient struct { + filer_pb.SeaweedFilerClient + dir string + nth int + listings int + pause func() +} + +func (c *pauseListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) { + if in.Directory == c.dir { + c.listings++ + if c.listings == c.nth { + c.pause() + } + } + return c.SeaweedFilerClient.ListEntries(ctx, in, opts...) +} + +func createNamespaceNsRequest(t *testing.T) *CreateNamespaceRequest { + return &CreateNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}} +} + +// A create could take a leftover directory back after the scan listed it, and a +// namespace with nothing in it holds no file for the walk to find, so the +// recursive delete would remove a namespace the create had already answered +// for. The delete's claim on the bucket turns the create away instead. +func TestDeleteTableBucketTurnsAwayATakeBackAfterTheScan(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "ns", nil) + var createStatus int + var createResp S3TablesError + // The scan lists the bucket again for the page after the leftover, so the + // create lands once the scan has recorded it and before the delete walks it. + client := &pauseListClient{SeaweedFilerClient: fs.Client, dir: bucketPath, nth: 2} + client.pause = func() { + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.GreaterOrEqual(t, client.listings, 2, "the create has to land after the scan listed the leftover") + assert.Equal(t, http.StatusConflict, createStatus, "%+v", createResp) + assert.Equal(t, ErrCodeConflict, createResp.Type, "%+v", createResp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") +} + +// A create that finds the bucket claimed by a delete is refused and writes +// nothing, as a create under a claimed namespace is. +func TestCreateNamespaceRefusedInClaimedBucket(t *testing.T) { + fs, m := startEmptyBucket(t) + plantDeleteClaim(fs, TablesPath, renameTestBucket) + + status, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetTableBucketPath(renameTestBucket), "ns"), "a refused create writes nothing") +} + +// A create is authorized against the bucket it read. If that bucket is deleted +// and another is created under the same name before the create writes, the +// namespace would land in a bucket nobody authorized it for, carrying the first +// bucket's owner. The create checks the bucket by its creation time, reads the +// new one as the old one gone, and writes nothing into it. +func TestCreateNamespaceTreatsARecreatedBucketAsGone(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + recreatedMeta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now(), + }) + require.NoError(t, err) + recreated := map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: recreatedMeta, + } + // The create looks its namespace up right before it writes; the bucket is + // replaced there. + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnLookup = nil + fs.Put(TablesPath, renameTestBucket, recreated) + } + + status, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchBucket, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written into the new bucket") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the new bucket stays") + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "and is left as it was created") +} + +// A delete is authorized against the bucket it read. If that bucket is deleted +// and another is created under the same name before the delete claims it, the +// claim and the removal would act on a bucket nobody authorized the delete for. +// The claim takes only the bucket with the creation time the delete read, so +// the new one reads as the old one gone and is left as it was. +func TestDeleteTableBucketTreatsARecreatedBucketAsGone(t *testing.T) { + fs, m := startEmptyBucket(t) + recreatedMeta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now(), + }) + require.NoError(t, err) + recreated := map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: recreatedMeta, + } + // The claim is the delete's first write. The bucket is replaced as it lands, + // so the write loses and its retry reads the new bucket. + fs.BeforeUpdate = func() { fs.Put(TablesPath, renameTestBucket, recreated) } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchBucket, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the new bucket stays") + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "and is left as it was created") +} + +// failNthLookupClient answers the nth lookup of target with Unavailable, the +// way a filer that is briefly unreachable would, and counts the lookups of the +// entries under target that follow. +type failNthLookupClient struct { + filer_pb.SeaweedFilerClient + target string + nth int + lookups int + childLookups int +} + +func (c *failNthLookupClient) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { + switch path.Join(req.Directory, req.Name) { + case c.target: + c.lookups++ + if c.lookups == c.nth { + return nil, status.Error(codes.Unavailable, "filer unavailable") + } + default: + if req.Directory == c.target { + c.childLookups++ + } + } + return c.SeaweedFilerClient.LookupDirectoryEntry(ctx, req, opts...) +} + +// The check of the bucket before the namespace is written either passes or ends +// the create. A bucket it could not read, or one it finds still being created, +// is not one to write into: nothing is written, where a create that went on +// would write and leave its confirmation to back out. +func TestCreateNamespaceStopsAtAFailedBucketCheck(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + + t.Run("the bucket could not be read", func(t *testing.T) { + fs, m := startEmptyBucket(t) + // The create reads the bucket three times for its authorization; the + // fourth read is the check before the write. + client := &failNthLookupClient{SeaweedFilerClient: fs.Client, target: bucketPath, nth: 4} + + code, resp := runHandler(t, m, client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusInternalServerError, code, "%+v", resp) + assert.Equal(t, ErrCodeInternalError, resp.Type, "%+v", resp) + assert.Zero(t, client.childLookups, "the create stops before it looks at the namespace") + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written") + }) + t.Run("the bucket is still being created", func(t *testing.T) { + fs, m := startEmptyBucket(t) + plantCreateMark(fs, TablesPath, renameTestBucket) + + code, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written") + }) +} + +// A delete that has scanned an empty bucket must not lose to a create that +// lands before the bucket is removed: the claim the delete holds since before +// the scan turns the create away, so no namespace is answered for and deleted. +func TestDeleteTableBucketTurnsAwayACreateAfterTheScan(t *testing.T) { + fs, m := startEmptyBucket(t) + var createStatus int + var createResp S3TablesError + fs.BeforeDelete = func() { + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createStatus, "the create has to land between the scan and the delete") + assert.Equal(t, http.StatusConflict, createStatus, "%+v", createResp) + assert.Equal(t, ErrCodeConflict, createResp.Type, "%+v", createResp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") +} + +// A create can pass the claim check before the delete claims the bucket, and +// then write a namespace that was not there when the delete scanned. The scan +// never saw it, and the recursive delete would remove it after the create had +// answered. The create's confirmation after its write finds the claim, so it +// backs out and reports the conflict instead of success. +func TestCreateNamespaceWrittenAfterTheScanBacksOut(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create looks its namespace up right after the claim check; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + // The delete claims the bucket and scans it empty. The create writes only + // now, before the delete removes the bucket. + var createResult handlerResult + fs.BeforeDelete = func() { + close(release) + createResult = <-created + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createResult.status, "the create has to write between the scan and the delete") + assert.Equal(t, http.StatusConflict, createResult.status, "%+v", createResult.resp) + assert.Equal(t, ErrCodeConflict, createResult.resp.Type, "%+v", createResult.resp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + assert.Nil(t, fs.Get(bucketPath, "ns"), "no namespace the create answered for is left behind") +} + +// The claim a delete takes on the bucket can be replaced by a second delete, +// which lifts it when it refuses. A create then finds the bucket open, and the +// first delete, which scanned before any of that, must not remove the bucket +// on a claim it no longer holds: it would take the namespace the create +// answered for, and the file under it, with it. It answers 409 and leaves both. +func TestDeleteTableBucketRefusesAfterItsClaimIsLifted(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + var refusedStatus, createStatus int + var refusedResp, createResp S3TablesError + // The first delete has scanned an empty bucket when it reaches its removal. + fs.BeforeDelete = func() { + // A directory with a file, kept by a namespace that was deleted, turns up + // and the second delete refuses on it, lifting its own claim. + fs.Put(bucketPath, "ns", nil) + fs.PutFile(nsPath, keptDataFile, time.Now()) + refusedStatus, refusedResp = runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + // The bucket is open again, so the create takes the directory back. + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createStatus, "the create has to land between the first delete's scan and its removal") + assert.Equal(t, http.StatusConflict, refusedStatus, "%+v", refusedResp) + assert.Equal(t, ErrCodeBucketNotEmpty, refusedResp.Type, "%+v", refusedResp) + assert.Equal(t, http.StatusOK, createStatus, "%+v", createResp) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting) + assert.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace the create answered for must survive") + assert.NotNil(t, fs.Get(nsPath, keptDataFile), "the file under it must survive") +} + +// The walk that finds a leftover directory empty is over before the bucket is +// removed, and a file can be written in between: an upload through the S3 API +// takes no claim. Nothing the removal does may take a file it was not shown, so +// it is refused at the directory that holds one, and a file written into the +// bucket itself makes the removal of the bucket refuse the same way. +func TestDeleteTableBucketKeepsAFileUploadedAfterTheWalk(t *testing.T) { + for _, tc := range []struct { + name string + dir func(bucketPath string) string + }{ + {"into a leftover directory", func(bucketPath string) string { return path.Join(bucketPath, "old") }}, + {"into the bucket", func(bucketPath string) string { return bucketPath }}, + } { + t.Run(tc.name, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + uploadDir := tc.dir(bucketPath) + fs.BeforeDelete = func() { + fs.PutFile(uploadDir, keptDataFile, time.Now()) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(uploadDir, keptDataFile), "the uploaded file must survive") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting, "a refused delete lifts its claim") + if uploadDir != bucketPath { + require.NotNil(t, fs.Get(bucketPath, "old")) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "and its claim on the leftover") + } + }) + } +} + +// A second delete can replace the first one's claim and lift it, and a create +// then takes a leftover directory back as a namespace after the first delete +// scanned it bare, and an empty table is created in it. Neither holds a file, +// so only the attributes the scan saw on the directory tell the removal it is +// no longer the same directory, and that has to be settled before anything +// under it goes: the delete refuses with a conflict and leaves both. +func TestDeleteTableBucketKeepsALeftoverTakenBackAfterItsClaimIsLifted(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + fs.Put(bucketPath, "ns", nil) + var refusedStatus, createStatus, tableStatus int + var refusedResp, createResp, tableResp S3TablesError + fs.BeforeDelete = func() { + // A file under another leftover makes the second delete refuse, and + // lift its claim; the bucket is open again for the create. + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + refusedStatus, refusedResp = runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + tableStatus, tableResp = runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, tableStatus, "the creates have to land between the first delete's scan and its removal") + assert.Equal(t, http.StatusConflict, refusedStatus, "%+v", refusedResp) + assert.Equal(t, http.StatusOK, createStatus, "%+v", createResp) + assert.Equal(t, http.StatusOK, tableStatus, "%+v", tableResp) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace the create answered for must survive") + assert.Contains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyMetadata) + require.NotNil(t, fs.Get(nsPath, "t"), "the table the create answered for must survive") + assert.Contains(t, fs.Get(nsPath, "t").Extended, ExtendedKeyMetadata) + assert.NotNil(t, fs.Get(path.Join(nsPath, "t"), "data"), "and so must its data directory") + assert.NotNil(t, fs.Get(path.Join(bucketPath, "old"), keptDataFile), "the file under the other leftover must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A catalog record under a leftover directory is not the leftover's to remove, +// whatever it holds: an empty table is still a table. The delete refuses at +// it, and lifts the claim it took on the leftover. +func TestDeleteTableBucketRefusesACatalogRecordInALeftover(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "t", map[string][]byte{ + ExtendedKeyMetadata: []byte("{}"), + ExtendedKeyEntryType: []byte(EntryTypeTable), + }) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(leftover, "t"), "the record must survive") + assert.Contains(t, fs.Get(leftover, "t").Extended, ExtendedKeyMetadata) + require.NotNil(t, fs.Get(bucketPath, "old")) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A namespace a create left marked unfinished must not wedge the name: the mark +// says its writer never confirmed it, so the next create takes it over, and +// tables can be created in it once it is confirmed. +func TestCreateNamespaceAdoptsAnUnfinishedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + plantCreateMark(fs, bucketPath, "ns") + + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "the adopted namespace ends confirmed") + requireOK(t, m, fs, "CreateTable", createTableRequest(t, "ns", "t")) +} + +// A table answers success once its namespace is open, and says nothing of the +// bucket, so a table created while the namespace's own create still has to +// confirm the bucket would survive as a table under a namespace that +// confirmation backs out. The namespace carries the unfinished mark until the +// confirmation has passed, and is absent to a create under it meanwhile. +func TestCreateTableRefusedInNamespaceStillBeingCreated(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create confirms the bucket right after writing the namespace; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != TablesPath || name != renameTestBucket || fs.Get(bucketPath, "ns") == nil { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + + status, resp := runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + close(release) + createResult := <-created + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(path.Join(bucketPath, "ns"), "t"), "a refused create writes nothing") + assert.Equal(t, http.StatusOK, createResult.status, "%+v", createResult.resp) + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "a confirmed namespace carries no mark") + requireOK(t, m, fs, "CreateTable", createTableRequest(t, "ns", "t")) +} + +// A namespace whose create has not confirmed it is absent to every reader, as +// a table's is. A delete that claimed it would replace the create's mark, and +// lifting its own claim on a refusal would leave the namespace looking +// confirmed: open to tables before its bucket check passed, and out of reach of +// the create's back-out. Here the create takes back a leftover holding a file, +// which a delete would refuse on. +func TestDeleteNamespaceLeavesAnUnfinishedNamespaceAlone(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + fs.Put(bucketPath, "ns", nil) + fs.PutFile(nsPath, keptDataFile, time.Now()) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create confirms the bucket right after writing the namespace; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != TablesPath || name != renameTestBucket { + return + } + if entry := fs.Get(bucketPath, "ns"); entry == nil || entry.Extended[ExtendedKeyMetadata] == nil { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + + deleteStatus, deleteResp := runHandler(t, m, fs.Client, "DeleteNamespace", + &DeleteNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}) + listed := listedNamespaces(t, m, fs) + tableStatus, tableResp := runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + close(release) + createResult := <-created + + assert.Equal(t, http.StatusNotFound, deleteStatus, "%+v", deleteResp) + assert.Equal(t, ErrCodeNoSuchNamespace, deleteResp.Type, "%+v", deleteResp) + assert.NotContains(t, listed, "ns", "an unfinished namespace is not listed") + assert.Equal(t, http.StatusNotFound, tableStatus, "%+v", tableResp) + assert.Equal(t, ErrCodeNoSuchNamespace, tableResp.Type, "%+v", tableResp) + assert.Nil(t, fs.Get(nsPath, "t"), "no table is created in it") + assert.Equal(t, http.StatusOK, createResult.status, "%+v", createResult.resp) + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "the create confirms it") + assert.NotNil(t, fs.Get(nsPath, keptDataFile), "the file under it stays") +} + +// A namespace can become unfinished between a delete's read and its claim: a +// create took its name back meanwhile. The claim refuses it as absent, rather +// than replacing the create's mark. +func TestDeleteNamespaceDoesNotClaimAnUnfinishedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + // The claim is the delete's first write. + fs.BeforeUpdate = func() { plantCreateMark(fs, bucketPath, "ns") } + + status, resp := runHandler(t, m, fs.Client, "DeleteNamespace", + &DeleteNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}) + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace stays") + assert.Equal(t, []byte(createClaim), fs.Get(bucketPath, "ns").Extended[ExtendedKeyDeleting], "with its create's mark") +} + +// A delete that claims the bucket between the namespace's confirmation and the +// end of its create must still see the namespace, or it would remove a +// namespace the create answered for. The mark does not hide it from the scan. +func TestDeleteTableBucketCountsANamespaceStillBeingCreated(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + plantCreateMark(fs, GetTableBucketPath(renameTestBucket), "ns") + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.Contains(t, resp.Message, "namespace ns", "the hidden namespace is named") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is kept") +} + +// The claim on the bucket is lifted whenever the delete refuses, or the bucket +// would refuse every create; a claim left by a delete that died does not keep +// the next delete from running. +func TestDeleteTableBucketClaimOnTheBucket(t *testing.T) { + lifted := func(t *testing.T, fs *s3tablestest.MemFiler, m *Manager) { + t.Helper() + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting) + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"other"}, + }) + } + t.Run("lifted when a namespace is found", func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + lifted(t, fs, m) + }) + t.Run("lifted when a directory holds files", func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + lifted(t, fs, m) + }) + t.Run("replaces the claim of a delete that died", func(t *testing.T) { + fs, m := startEmptyBucket(t) + plantDeleteClaim(fs, TablesPath, renameTestBucket) + + requireOK(t, m, fs, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + }) +} + +// putDirectoryObject stores an S3 object with content on the directory entry +// dir/name, the way the S3 API stores a key that other keys sit under: with the +// prefix-object mark the filer expects on an object landing on a directory. +func putDirectoryObject(t *testing.T, fs *s3tablestest.MemFiler, dir, name string) { + t.Helper() + _, err := fs.Client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{ + Directory: dir, + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{Mime: "application/octet-stream"}, + Content: []byte("object"), + Extended: map[string][]byte{s3_constants.SeaweedFSPrefixObject: []byte("true")}, + }, + }) + require.NoError(t, err) +} + +// markPrefixObject marks the directory dir/name as also holding an empty S3 +// object, which leaves no trace but the mark, keeping its other attributes. +func markPrefixObject(fs *s3tablestest.MemFiler, dir, name string) { + extended := map[string][]byte{} + for key, value := range fs.Get(dir, name).Extended { + extended[key] = value + } + extended[s3_constants.SeaweedFSPrefixObject] = []byte("true") + fs.Put(dir, name, extended) +} + +// beforeDeleteClient runs before just ahead of the first delete of target. +type beforeDeleteClient struct { + filer_pb.SeaweedFilerClient + target string + before func() +} + +func (c *beforeDeleteClient) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) { + if before := c.before; before != nil && path.Join(req.Directory, req.Name) == c.target { + c.before = nil + before() + } + return c.SeaweedFilerClient.DeleteEntry(ctx, req, opts...) +} + +// An S3 object can be stored on a directory entry: a key other keys sit under, +// or an empty object that leaves nothing but a mark on the directory. Such a +// directory holds no file, yet it is user data, not a leftover: the delete +// refuses wherever it finds one and the object stays. +func TestDeleteTableBucketKeepsADirectoryBackedObject(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + refused := func(t *testing.T, code int, resp S3TablesError, wantType string) { + t.Helper() + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, wantType, resp.Type, "%+v", resp) + } + + t.Run("an empty object in the bucket", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", map[string][]byte{s3_constants.SeaweedFSPrefixObject: []byte("true")}) + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + }) + t.Run("an object under a leftover directory", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + putDirectoryObject(t, fs, leftover, "a") + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(leftover, "a"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(leftover, "a").Content) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a leftover stored as an object after the scan", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.BeforeDelete = func() { putDirectoryObject(t, fs, bucketPath, "old") } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(bucketPath, "old").Content) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "no claim is taken on an object") + }) + t.Run("a leftover stored as an object between the read and the write of its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + // The claim read the bare directory; the object lands before its write, + // which would otherwise put the bare directory back over it. + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "old" { + return + } + fs.OnUpdate = nil + putDirectoryObject(t, fs, bucketPath, "old") + } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(bucketPath, "old").Content) + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "no claim is taken on an object") + }) + t.Run("a leftover marked as an empty object after the walk", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + client := &beforeDeleteClient{SeaweedFilerClient: fs.Client, target: leftover} + client.before = func() { markPrefixObject(fs, bucketPath, "old") } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeConflict) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + }) +} + +// A directory the delete listed can be removed by another client and a file +// uploaded at its path before the delete reaches it. The file carries no +// attribute a directory would not, so only the directory's type and the claim +// written onto it tell the two apart: the delete refuses and the file stays, +// whether it replaced the leftover itself or a directory under it, and whether +// it landed before that directory was claimed or after. +func TestDeleteTableBucketKeepsAFileThatReplacedADirectory(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + kept := func(t *testing.T, fs *s3tablestest.MemFiler, dir, name string) { + t.Helper() + entry := fs.Get(dir, name) + require.NotNil(t, entry, "the uploaded file must survive") + assert.False(t, entry.IsDirectory) + assert.NotContains(t, entry.Extended, ExtendedKeyDeleting, "no claim is left on the file") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + } + + t.Run("the leftover, after the scan", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.BeforeDelete = func() { fs.PutFile(bucketPath, "old", time.Now()) } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, bucketPath, "old") + }) + t.Run("a directory under it, after the walk listed it", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + // The scan's look for files and the removal's walk each list old, then + // old/child. The file takes child's place as the walk reaches it, after + // the walk listed child as a directory and before child is claimed. + client := &pauseListClient{SeaweedFilerClient: fs.Client, dir: path.Join(leftover, "child"), nth: 2} + client.pause = func() { fs.PutFile(leftover, "child", time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.Equal(t, 2, client.listings, "the file has to land during the removal's walk") + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a directory under it, between the read and the write of its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + // The filer refuses to write the claimed directory over the file and + // says so only as an opaque failure. + fs.OnUpdate = func(dir, name string) { + if dir != leftover || name != "child" { + return + } + fs.OnUpdate = nil + fs.PutFile(leftover, "child", time.Now()) + } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a directory under it, after its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + client := &beforeDeleteClient{SeaweedFilerClient: fs.Client, target: path.Join(leftover, "child")} + client.before = func() { fs.PutFile(leftover, "child", time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.Nil(t, client.before, "the file has to land right before the delete") + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) +} + +// failClaimReleaseClient answers the first write that lifts a delete claim from +// target with Unavailable, the way a filer that is briefly unreachable would. +type failClaimReleaseClient struct { + filer_pb.SeaweedFilerClient + target string + failed int +} + +func (c *failClaimReleaseClient) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) { + if c.failed == 0 && req.Entry != nil && path.Join(req.Directory, req.Entry.Name) == c.target && + req.Entry.Extended[ExtendedKeyDeleting] == nil && len(req.ExpectedExtended[ExtendedKeyDeleting]) > 0 { + c.failed++ + return nil, status.Error(codes.Unavailable, "filer unavailable") + } + return c.SeaweedFilerClient.UpdateEntry(ctx, req, opts...) +} + +// A claim the delete fails to lift refuses every create in the bucket until the +// next delete replaces it, so a release the filer turns away once is retried. +func TestDeleteTableBucketRetriesTheClaimRelease(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + client := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: bucketPath} + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, client.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) +} + +// A claim the removal fails to lift from a leftover directory makes a create +// read the leftover's name as taken, and one left on a directory under it counts +// as a child of the namespace the leftover may become. A release the filer turns +// away once is retried for both. +func TestDeleteTableBucketRetriesLeftoverClaimReleases(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + + t.Run("on the leftover", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + failing := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: leftover} + // A file lands in the leftover after the scan, so the walk refuses it and + // the leftover's claim has to come off. + client := &beforeDeleteClient{SeaweedFilerClient: failing, target: GetTableObjectBucketPath(renameTestBucket)} + client.before = func() { fs.PutFile(leftover, keptDataFile, time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, failing.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"old"}}) + }) + t.Run("on a directory under it", func(t *testing.T) { + fs, m := startEmptyBucket(t) + child := path.Join(leftover, "child") + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + failing := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: child} + // A file lands in child after its claim, so its delete is refused and + // the claim has to come off. + client := &beforeDeleteClient{SeaweedFilerClient: failing, target: child} + client.before = func() { fs.PutFile(child, keptDataFile, time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, failing.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(leftover, "child").Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + assert.NotNil(t, fs.Get(child, keptDataFile), "the file stays") + }) +} + +// The scan reads the bucket a page at a time, and the leftovers it collects +// along the way do not decide anything. With more bare directories than one page +// holds, the namespace sorted after them is on a later page: stopping at the +// first page would find nothing but leftovers and delete the bucket with it. +func TestDeleteTableBucketFindsANamespacePastLeftovers(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + for i := 0; i <= 10; i++ { + fs.Put(bucketPath, fmt.Sprintf("a%02d", i), nil) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"zz"}, + }) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(bucketPath, "zz"), "the namespace must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} diff --git a/weed/s3api/s3tables/handler_delete_create_race_test.go b/weed/s3api/s3tables/handler_delete_create_race_test.go index 38f075d3a9..b46b9fb8d0 100644 --- a/weed/s3api/s3tables/handler_delete_create_race_test.go +++ b/weed/s3api/s3tables/handler_delete_create_race_test.go @@ -647,6 +647,22 @@ func TestDeleteNamespaceWithTableLiftsItsClaim(t *testing.T) { } } +// A claim the delete fails to lift refuses every create in the namespace, so a +// release the filer turns away once is retried. +func TestDeleteNamespaceRetriesTheClaimRelease(t *testing.T) { + fs, m := startRenameManager(t) + bucketPath := GetTableBucketPath(renameTestBucket) + before := fs.Get(bucketPath, "ns").Extended + client := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: GetNamespacePath(renameTestBucket, "ns")} + + code, resp := runHandler(t, m, client, "DeleteNamespace", deleteNamespaceRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeNamespaceNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, client.failed, "the first release has to fail") + assert.Equal(t, before, fs.Get(bucketPath, "ns").Extended, "a later attempt lifts the claim") +} + // orphanMarkedRecord plants what a create that crashed after writing its // record leaves behind: catalog attributes under the unfinished mark. func orphanMarkedRecord(fs *s3tablestest.MemFiler, dir, name string) { diff --git a/weed/s3api/s3tables/handler_namespace.go b/weed/s3api/s3tables/handler_namespace.go index a4690f491a..2683e06fd0 100644 --- a/weed/s3api/s3tables/handler_namespace.go +++ b/weed/s3api/s3tables/handler_namespace.go @@ -170,20 +170,50 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R } namespacePath := GetNamespacePath(bucketName, namespaceName) + // The authorization above holds for this bucket only. One deleted and + // created again under the same name since then is another bucket, and is + // treated as gone by both checks below. + isAuthorizedBucket := isTableBucketCreatedAt(bucketMetadata.CreatedAt) - // Check if namespace already exists + // A delete that has claimed the bucket is scanning it for namespaces, so + // refuse before writing anything. Checked after the authorization above, so + // denied callers never see the delete. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - _, err := h.readNamespaceMetadataAttr(r.Context(), client, namespacePath) + return h.checkParentOpen(r.Context(), client, bucketPath, isAuthorizedBucket) + }) + if h.writeBucketClosedError(w, err, bucketName) { return err + } + if err != nil { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err)) + return err + } + + // Check if namespace already exists. One still marked unfinished does not: + // its create either confirmed it or lost it, and the mark left behind would + // wedge the name, so this create takes it over the way a table's does. + exists := false + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + entry, err := h.lookupEntry(r.Context(), client, namespacePath) + if errors.Is(err, filer_pb.ErrNotFound) { + return nil + } + if err != nil { + return err + } + _, hasMetadata := entry.Extended[ExtendedKeyMetadata] + exists = hasMetadata && !isUnfinishedCreate(entry.Extended) + return nil }) - if err == nil { - h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName)) - return fmt.Errorf("namespace already exists") - } else if !errors.Is(err, filer_pb.ErrNotFound) { + if err != nil { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err)) return err } + if exists { + h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName)) + return fmt.Errorf("namespace already exists") + } // Create the namespace with bucket owner to maintain consistency // (authorization above ensures the caller has permission to create in this bucket) @@ -201,7 +231,13 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return fmt.Errorf("failed to marshal metadata: %w", err) } - attrs := map[string][]byte{ExtendedKeyMetadata: metadataBytes} + // The unfinished mark keeps a table or view from being created in the + // namespace until the bucket check below has passed: one created before it + // would be left under a namespace that check then backs out. + attrs := map[string][]byte{ + ExtendedKeyMetadata: metadataBytes, + ExtendedKeyDeleting: []byte(createClaim), + } var taken error err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { err := h.createDirectoryExclusive(r.Context(), client, namespacePath, attrs) @@ -215,7 +251,19 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return nil } } - return err + if err != nil { + return err + } + // A delete may have claimed the bucket since the check above, after it + // scanned for namespaces and before this write. Confirm the bucket once + // more now that the namespace is there: a delete claiming it later scans + // the namespace, marked or not, and one that claimed it already makes + // this back out what it wrote. The mark comes off only after that check + // and the bucket is not checked again: a table created in the window + // between the two would be left under a namespace the second check backs out. + // A bucket recreated since the authorization fails the check as gone, so + // the namespace is backed out of it rather than left with this owner. + return h.confirmParentOnce(r.Context(), client, bucketPath, isAuthorizedBucket, namespacePath, attrs, namespacePath) }) if err == nil && taken != nil { err = taken @@ -228,6 +276,9 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R h.writeError(w, http.StatusConflict, ErrCodeConflict, "namespace changed during create, retry the request") return err } + if h.writeBucketClosedError(w, err, bucketName) { + return err + } if err != nil { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace") return err @@ -243,6 +294,25 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return nil } +// writeBucketClosedError answers a create refused because its bucket is being +// created, being deleted, or was deleted, and reports whether err was one of +// those. +func (h *S3TablesHandler) writeBucketClosedError(w http.ResponseWriter, err error, bucketName string) bool { + switch { + case errors.Is(err, errParentCreating): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("table bucket %s is still being created, retry the request", bucketName)) + case errors.Is(err, errParentDeleting): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("table bucket %s is being deleted", bucketName)) + case errors.Is(err, errParentGone): + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + case errors.Is(err, errBackOutFailed): + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("table bucket %s was deleted during the request and the new namespace may remain", bucketName)) + default: + return false + } + return true +} + // handleGetNamespace gets details of a namespace func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error { var req GetNamespaceRequest @@ -580,9 +650,10 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re continue } - // Read metadata from extended attribute + // Read metadata from extended attribute. A namespace its create has + // not confirmed is not one yet. data, ok := entry.Entry.Extended[ExtendedKeyMetadata] - if !ok { + if !ok || isUnfinishedCreate(entry.Entry.Extended) { continue } @@ -718,11 +789,14 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R } // Claim before scanning, so a create that writes after the scan sees it. + // A namespace whose create has not confirmed it is absent, and claiming it + // would replace the create's mark: a refused delete then lifts the claim and + // leaves the namespace looking confirmed before its create checked the bucket. var claim string var claimed map[string][]byte err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { var claimErr error - claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isNamespaceEntry) + claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isFinishedNamespaceEntry) return claimErr }) if err != nil { diff --git a/weed/s3api/s3tables/handler_table.go b/weed/s3api/s3tables/handler_table.go index 8e6ce93c96..a9df3373d8 100644 --- a/weed/s3api/s3tables/handler_table.go +++ b/weed/s3api/s3tables/handler_table.go @@ -2270,9 +2270,12 @@ func (h *S3TablesHandler) authorizeCreateTable(w http.ResponseWriter, r *http.Re } // writeNamespaceClosedError answers a create refused because its namespace is -// being deleted or was deleted, and reports whether err was one of those. +// being created, being deleted, or was deleted, and reports whether err was one +// of those. func (h *S3TablesHandler) writeNamespaceClosedError(w http.ResponseWriter, err error, namespaceName string) bool { switch { + case errors.Is(err, errParentCreating): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("namespace %s is still being created, retry the request", namespaceName)) case errors.Is(err, errParentDeleting): h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("namespace %s is being deleted", namespaceName)) case errors.Is(err, errParentGone), errors.Is(err, filer_pb.ErrNotFound): @@ -2285,13 +2288,14 @@ func (h *S3TablesHandler) writeNamespaceClosedError(w http.ResponseWriter, err e return true } -// refuseClaimedNamespace answers 409 when a delete has claimed the namespace. -// Call it after the authorization check, so denied callers never see the delete. +// refuseClaimedNamespace answers 409 when a delete has claimed the namespace or +// its create is not confirmed yet. Call it after the authorization check, so +// denied callers never see the delete. func (h *S3TablesHandler) refuseClaimedNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient, namespacePath, namespaceName string) error { err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.checkParentOpen(r.Context(), client, namespacePath, isNamespaceEntry) }) - if errors.Is(err, errParentDeleting) { + if errors.Is(err, errParentDeleting) || errors.Is(err, errParentCreating) { h.writeNamespaceClosedError(w, err, namespaceName) return err } diff --git a/weed/s3api/s3tables/s3tablestest/memfiler.go b/weed/s3api/s3tables/s3tablestest/memfiler.go index 391a2f0331..292a7c94cd 100644 --- a/weed/s3api/s3tables/s3tablestest/memfiler.go +++ b/weed/s3api/s3tables/s3tablestest/memfiler.go @@ -204,12 +204,10 @@ func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryReque } f.mu.Lock() defer f.mu.Unlock() - // The real filer refuses a non-recursive delete of a non-empty folder. - if child := path.Join(req.Directory, req.Name); !req.IsRecursive && len(f.entries[child]) > 0 { - return &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder: " + child}, nil - } // The real filer evaluates ExpectedExtended under the path lock and refuses - // the delete when the entry no longer matches what the caller captured. + // the delete when the entry no longer matches what the caller captured. It + // does so before anything else, so a directory that changed hands is a + // changed entry whether or not it also holds something. for key, expected := range req.ExpectedExtended { var actual []byte if d, ok := f.entries[req.Directory]; ok { @@ -221,6 +219,10 @@ func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryReque return nil, status.Errorf(codes.FailedPrecondition, "extended attribute %q changed", key) } } + // The real filer then refuses a non-recursive delete of a non-empty folder. + if child := path.Join(req.Directory, req.Name); !req.IsRecursive && len(f.entries[child]) > 0 { + return &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder: " + child}, nil + } if d, ok := f.entries[req.Directory]; ok { delete(d, req.Name) } diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 9b3704aa60..d647668b18 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -43,6 +43,10 @@ var handlers = map[string]func(*Server, http.ResponseWriter, *http.Request){ // lance table ids put indexes under a "vector" namespace inside their bucket. const indexNamespace = s3tables.VectorIndexNamespace +// ensureIndexNamespaceAttempts bounds how often ensureIndexNamespace declares +// the namespace again after losing it to a concurrent declare. +const ensureIndexNamespaceAttempts = 3 + // Vector bucket records are s3tables table buckets; the S3 Vectors ARN form is // arn:aws:s3vectors:{region}:{account}:bucket/{name}[/index/{name}]. func (s *Server) vectorBucketArn(bucket string) string { @@ -220,26 +224,36 @@ func vectorErr(w http.ResponseWriter, err error) { // calls are internal: a caller's CreateTable grant is what authorizes the // index, so it must not also need namespace reads or writes. func (s *Server) ensureIndexNamespace(ctx context.Context, bucket string) error { - var resp map[string]interface{} - err := s.internalCatalog(ctx, "GetNamespace", &s3tables.GetNamespaceRequest{ - TableBucketARN: s.s3tablesArn(bucket), - Namespace: []string{indexNamespace}, - }, &resp) - if err == nil { - return nil + var err error + for attempt := 0; attempt < ensureIndexNamespaceAttempts; attempt++ { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 50 * time.Millisecond) + } + var resp map[string]interface{} + err = s.internalCatalog(ctx, "GetNamespace", &s3tables.GetNamespaceRequest{ + TableBucketARN: s.s3tablesArn(bucket), + Namespace: []string{indexNamespace}, + }, &resp) + if err == nil { + return nil + } + if code := catalogErrCode(err); code != s3tables.ErrCodeNoSuchNamespace && code != s3tables.ErrCodeNoSuchBucket { + return err + } + err = s.internalCatalog(ctx, "CreateNamespace", &s3tables.CreateNamespaceRequest{ + TableBucketARN: s.s3tablesArn(bucket), + Namespace: []string{indexNamespace}, + }, &resp) + // A concurrent CreateIndex may have won the create, or taken over this + // one's namespace before it was confirmed. + if err == nil || catalogErrCode(err) == s3tables.ErrCodeNamespaceAlreadyExists { + return nil + } + if catalogErrCode(err) != s3tables.ErrCodeConflict { + return err + } } - if code := catalogErrCode(err); code != s3tables.ErrCodeNoSuchNamespace && code != s3tables.ErrCodeNoSuchBucket { - return err - } - err = s.internalCatalog(ctx, "CreateNamespace", &s3tables.CreateNamespaceRequest{ - TableBucketARN: s.s3tablesArn(bucket), - Namespace: []string{indexNamespace}, - }, &resp) - // A concurrent CreateIndex may have won the create. - if err != nil && catalogErrCode(err) != s3tables.ErrCodeNamespaceAlreadyExists { - return err - } - return nil + return err } func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request) { diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index 7aad7447bb..32dbef2d70 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "testing" "time" @@ -983,3 +984,55 @@ func TestVectorKeysBoundsRejectsBeforeEngine(t *testing.T) { t.Fatalf("engine saw %d get/%d delete calls, want none", len(f.gets), len(f.deletes)) } } + +// Two indexes created at once in a new bucket both declare the implicit +// namespace. One create takes over the other's namespace while it is still +// marked unfinished, so the other's confirmation fails; its index is created +// all the same. +func TestCreateIndexSurvivesAConcurrentNamespaceCreate(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + bucketPath := s3tables.GetTableBucketPath("vec-bucket") + reached := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseFirst := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseFirst) + held := false + // Hold the first create where it confirms the bucket after writing the namespace. + fs.OnLookup = func(dir, name string) { + if held || dir != s3tables.TablesPath || name != "vec-bucket" { + return + } + ns := fs.Get(bucketPath, indexNamespace) + if ns == nil || string(ns.Extended[s3tables.ExtendedKeyDeleting]) != "create" { + return + } + held = true + close(reached) + <-release + } + createIndex := func(name string) *httptest.ResponseRecorder { + return postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: name, DataType: "float32", Dimension: 2, DistanceMetric: "cosine", + }) + } + first := make(chan *httptest.ResponseRecorder, 1) + go func() { first <- createIndex("first") }() + select { + case <-reached: + case <-time.After(10 * time.Second): + t.Fatal("the first create never wrote an unfinished namespace") + } + + if w := createIndex("second"); w.Code != http.StatusOK { + t.Fatalf("second create index: %d %s", w.Code, w.Body.String()) + } + releaseFirst() + if w := <-first; w.Code != http.StatusOK { + t.Fatalf("first create index: %d %s", w.Code, w.Body.String()) + } +} From ab204a03e60f61c7293ba62ae939c9a7d540deb1 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Mon, 2 Nov 2026 05:57:45 +0530 Subject: [PATCH 02/31] fix(s3): verify SigV4 signature with decoded path behind X-Forwarded-Prefix (#12299) * fix(s3): verify SigV4 signature with decoded path behind X-Forwarded-Prefix When SeaweedFS is hosted behind a reverse proxy that strips a URL prefix (X-Forwarded-Prefix) and forwards the percent-encoded request path, signature verification fails for object keys containing non-ASCII characters. The client signs the canonical URI as the single percent-encoded path including the prefix, but the server verifies against the already-encoded forwarded path joined to the prefix, producing a mismatch (SignatureDoesNotMatch). Decode the forwarded path and rejoin the prefix before verifying, mirroring the existing un-prefixed decoded retry. ASCII keys are unaffected because the decoded path equals the original, so the retry is skipped. Signed-off-by: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> * s3: build the SigV4 candidate paths once and try them per host The forwarded-prefix and plain branches each repeated the decode-and-verify step for every host candidate. List the escaped and decoded paths once, the prefixed forms first, and verify them in the same order as before. Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- weed/s3api/auth_signature_v4.go | 40 ++++++++++++---------------- weed/s3api/auth_signature_v4_test.go | 33 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 5f96669554..661bd7002f 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -374,35 +374,29 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh if trustedPeer { forwardedPrefix = r.Header.Get("X-Forwarded-Prefix") } + // 9. Candidate paths: the escaped path as sent, then the decoded path that + // an S3 client encodes once when signing; each with the forwarded prefix first. + signaturePaths := []string{pathForSignature} + if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature { + signaturePaths = append(signaturePaths, decodedPath) + } + if forwardedPrefix != "" { + prefixedPaths := make([]string, 0, 2*len(signaturePaths)) + for _, p := range signaturePaths { + prefixedPaths = append(prefixedPaths, buildPathWithForwardedPrefix(forwardedPrefix, p)) + } + signaturePaths = append(prefixedPaths, signaturePaths...) + } var matchedHost string +hostCandidates: for i, hostCandidate := range extractHostHeaderCandidates(r, iam.externalHost) { if i > 0 && !replaceSignedHostHeader(extractedSignedHeaders, hostCandidate) { break } - - // 9. Verify with the X-Forwarded-Prefix path first - if forwardedPrefix != "" { - cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, pathForSignature) - calculatedSignature, errCode = verify(cleanedPath) - if errCode == s3err.ErrNone { + for _, p := range signaturePaths { + if calculatedSignature, errCode = verify(p); errCode == s3err.ErrNone { matchedHost = hostCandidate - break - } - } - - // 10. Verify with the original path - calculatedSignature, errCode = verify(pathForSignature) - if errCode == s3err.ErrNone { - matchedHost = hostCandidate - break - } - - // 11. Retry with decoded path if signature used raw path encoding - if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature { - calculatedSignature, errCode = verify(decodedPath) - if errCode == s3err.ErrNone { - matchedHost = hostCandidate - break + break hostCandidates } } } diff --git a/weed/s3api/auth_signature_v4_test.go b/weed/s3api/auth_signature_v4_test.go index a426357b02..e5abf46606 100644 --- a/weed/s3api/auth_signature_v4_test.go +++ b/weed/s3api/auth_signature_v4_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "net/http" + "net/url" "slices" "sync" "testing" @@ -735,3 +736,35 @@ func TestVerifyV4SignatureUsesOriginalRequestPath(t *testing.T) { t.Fatalf("signed request rejected after path rewrite: %d", errCode) } } + +// TestSignatureV4WithForwardedPrefixUnicodeKey guards against a signature +// mismatch when an object key contains non-ASCII characters and the request is +// received through a trusted reverse proxy that strips a path prefix. The +// client signs the single percent-encoded path including the prefix, so the +// server must decode the forwarded (already encoded) path before re-encoding it +// together with the prefix. +func TestSignatureV4WithForwardedPrefixUnicodeKey(t *testing.T) { + iam := newTestIAMWithTrustedProxy() + + bucket := "test-bucket" + key := "たあいまd·.jpg" + + // The proxy strips the X-Forwarded-Prefix and forwards the percent-encoded + // object path (the server sees the encoded form, not the raw Unicode key). + r, err := newTestRequest("GET", "https://example.com/"+bucket+"/"+url.PathEscape(key), 0, nil) + if err != nil { + t.Fatalf("failed to create test request: %v", err) + } + + r.Header.Set("X-Forwarded-Prefix", "/s3") + r.Header.Set("Host", "example.com") + r.RemoteAddr = "10.0.0.1:1234" + + // The client signs the full URL path including the prefix, using the single + // percent-encoded form of the Unicode key. + signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "/s3/"+bucket+"/"+key) + + if _, _, errCode := iam.doesSignatureMatch(r); errCode != s3err.ErrNone { + t.Fatalf("signature verification failed for unicode key behind forwarded prefix: code %d", int(errCode)) + } +} From 5cf053c32e3b270a3a8591b9f4bef05f233f6730 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 16:31:22 -0800 Subject: [PATCH 03/31] volume: read a manifest chunk from another replica when one fails (#12313) * operation: read a manifest chunk from the next replica when one fails ChunkedFileReader read each chunk from one random location and returned the first error, so a chunk whose cached replica went down failed the whole file although another replica had it. Try the chunk's locations in turn (local-data replicas first) and resume each attempt at the byte the previous one reached. A failed write to the caller still ends the read. Co-Authored-By: Claude Opus 5.5 * operation: drop a chunk volume's cached locations after a failed read A chunk read that failed on a replica left that replica in the 10-minute location cache, so later reads kept trying it first. Drop the volume's cached locations after a failure, and when every cached replica has failed, ask the master once more and try the locations it now lists. Co-Authored-By: Claude Opus 5.5 * operation: don't continue a partly sent chunk on another replica A volume server streams a large needle before it has verified the checksum and aborts the response on a mismatch, so bytes that already reached the reader may be corrupt. Continuing such a chunk on another replica would hand back a mix of bad and good bytes as a complete read. Move to the next replica only while nothing of the chunk has been written; a mid-body failure still fails the read, and the dropped cache entry lets the next read pick a live replica. Co-Authored-By: Claude Opus 5.5 * operation: treat a short chunk body as a failed replica read A replica that ends a chunk's body cleanly before the chunk's size made the read look successful, so the next replica was never tried and the following chunks landed at the wrong offsets. Read exactly the chunk's remaining length and report a short body as an error. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/operation/chunked_file.go | 78 +++++++-- weed/operation/chunked_file_test.go | 257 ++++++++++++++++++++++++++++ weed/operation/lookup.go | 34 ++-- 3 files changed, 342 insertions(+), 27 deletions(-) create mode 100644 weed/operation/chunked_file_test.go diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index a5746c8462..5df1647bc1 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "sort" + "strings" "sync" "google.golang.org/grpc" @@ -94,7 +95,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g return nil } -func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) { +func readChunkNeedle(fileUrl string, w io.Writer, offset int64, size int64, jwt string) (written int64, e error) { req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return written, err @@ -125,7 +126,11 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri return written, util_http.StatusError(fileUrl, resp) } - return io.Copy(w, resp.Body) + written, e = io.CopyN(w, resp.Body, size) + if e == io.EOF { + e = fmt.Errorf("%s: %d of %d bytes: %w", fileUrl, written, size, io.ErrUnexpectedEOF) + } + return written, e } func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader { @@ -175,26 +180,69 @@ func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) { return n, ErrInvalidRange } for ; chunkIndex < len(cf.chunkList); chunkIndex++ { - ci := cf.chunkList[chunkIndex] - // if we need read date from local volume server first? - fileUrl, jwt, lookupError := LookupFileId(func(_ context.Context) pb.ServerAddress { - return cf.master - }, cf.grpcDialOption, ci.Fid) - if lookupError != nil { - return n, lookupError - } - if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset, jwt); e != nil { + wn, e := cf.readChunk(cf.chunkList[chunkIndex], w, chunkStartOffset) + n += wn + cf.pos += wn + if e != nil { return n, e - } else { - n += wn - cf.pos += wn } - chunkStartOffset = 0 } return n, nil } +// readChunk reads the chunk from the first replica that serves it. A failure +// drops the volume's cached locations, and once every cached replica has +// failed the master is asked again. A replica that already sent part of the +// chunk is not continued elsewhere: a volume server streams a needle before +// its checksum is verified, so those bytes may be the corrupt ones. +func (cf *ChunkedFileReader) readChunk(ci *ChunkInfo, w io.Writer, offset int64) (written int64, err error) { + masterFn := func(_ context.Context) pb.ServerAddress { + return cf.master + } + vid, _, _ := strings.Cut(ci.Fid, ",") + dst := &writeErrorRecorder{Writer: w} + tried := make(map[string]bool) + for range 2 { + fileUrls, jwt, lookupError := lookupFileIdUrls(masterFn, cf.grpcDialOption, ci.Fid) + if lookupError != nil { + if err == nil { + err = lookupError + } + return 0, err + } + for _, fileUrl := range fileUrls { + if tried[fileUrl] { + continue + } + tried[fileUrl] = true + written, err = readChunkNeedle(fileUrl, dst, offset, ci.Size-offset, jwt) + if err == nil || dst.err != nil { + return written, err + } + glog.V(0).Infof("read chunk %s from %s: %v", ci.Fid, fileUrl, err) + InvalidateVolumeIdLocationCache(vid) + if written > 0 { + return written, err + } + } + } + return 0, err +} + +type writeErrorRecorder struct { + io.Writer + err error +} + +func (r *writeErrorRecorder) Write(p []byte) (int, error) { + n, err := r.Writer.Write(p) + if err != nil { + r.err = err + } + return n, err +} + func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) { cf.Seek(off, 0) return cf.Read(p) diff --git a/weed/operation/chunked_file_test.go b/weed/operation/chunked_file_test.go new file mode 100644 index 0000000000..589f77f16d --- /dev/null +++ b/weed/operation/chunked_file_test.go @@ -0,0 +1,257 @@ +package operation + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type fakeVolumeLocationServer struct { + master_pb.UnimplementedSeaweedServer + mu sync.Mutex + calls int + urls []string +} + +func (s *fakeVolumeLocationServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + resp := &master_pb.LookupVolumeResponse{} + for _, vid := range req.VolumeOrFileIds { + var locs []*master_pb.Location + for _, u := range s.urls { + locs = append(locs, &master_pb.Location{Url: u}) + } + resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ + VolumeOrFileId: vid, + Locations: locs, + }) + } + return resp, nil +} + +func (s *fakeVolumeLocationServer) lookups() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +type chunkReplica struct { + *httptest.Server + mu sync.Mutex + ranges []string +} + +func (c *chunkReplica) host() string { + return strings.TrimPrefix(c.URL, "http://") +} + +func (c *chunkReplica) requests() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.ranges...) +} + +// newChunkReplica serves data; with cutAfter >= 0 it announces the full length +// but drops the connection after cutAfter bytes. +func newChunkReplica(t *testing.T, data []byte, cutAfter int) *chunkReplica { + c := &chunkReplica{} + c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.mu.Lock() + c.ranges = append(c.ranges, r.Header.Get("Range")) + c.mu.Unlock() + if cutAfter < 0 { + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data)) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(http.StatusOK) + w.Write(data[:cutAfter]) + w.(http.Flusher).Flush() + conn, _, err := w.(http.Hijacker).Hijack() + if err == nil { + conn.Close() + } + })) + t.Cleanup(c.Close) + return c +} + +func deadAddress(t *testing.T) string { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + addr := lis.Addr().String() + lis.Close() + return addr +} + +func testChunkData(size int) []byte { + data := make([]byte, size) + for i := range data { + data[i] = byte(i * 7) + } + return data +} + +func TestChunkedFileReaderTriesTheNextReplica(t *testing.T) { + const vid = "91201" + data := testChunkData(4096) + healthy := newChunkReplica(t, data, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + vc.Set(vid, []Location{{Url: deadAddress(t)}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, master, dial) + reader.Seek(1000, io.SeekStart) + got, err := io.ReadAll(reader) + reader.Close() + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got, data[1000:]) { + t.Fatalf("read %d bytes, want the last %d bytes of the chunk", len(got), len(data)-1000) + } + if r := healthy.requests(); len(r) != 1 || r[0] != "bytes=1000-" { + t.Fatalf("healthy replica requests = %q, want one from byte 1000", r) + } + if _, err := vc.Get(vid); err == nil { + t.Fatalf("cached locations kept after a replica failed") + } + if fake.lookups() != 0 { + t.Fatalf("master lookups = %d, want 0", fake.lookups()) + } +} + +func TestChunkedFileReaderDoesNotContinueAPartlySentChunk(t *testing.T) { + const vid = "91205" + data := testChunkData(4096) + broken := newChunkReplica(t, data, 1000) + healthy := newChunkReplica(t, data, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + vc.Set(vid, []Location{{Url: broken.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, master, dial) + _, err := io.ReadAll(reader) + reader.Close() + if err == nil { + t.Fatalf("read of a chunk cut off mid-body succeeded") + } + if r := healthy.requests(); len(r) != 0 { + t.Fatalf("healthy replica requests = %q, want none", r) + } + + reader = NewChunkedFileReader(chunks, master, dial) + got, err := io.ReadAll(reader) + reader.Close() + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("second read: %d bytes, err %v", len(got), err) + } + if fake.lookups() != 1 || len(broken.requests()) != 1 { + t.Fatalf("second read: %d master lookups, %d requests to the broken replica; want 1 and 1", fake.lookups(), len(broken.requests())) + } +} + +func TestChunkedFileReaderAsksMasterWhenCachedReplicasFail(t *testing.T) { + const vid = "91202" + first, second := testChunkData(3000), testChunkData(2000) + healthy := newChunkReplica(t, second, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + firstReplica := newChunkReplica(t, first, -1) + const otherVid = "91203" + vc.Set(otherVid, []Location{{Url: firstReplica.host()}}, time.Minute) + vc.Set(vid, []Location{{Url: deadAddress(t)}}, time.Minute) + t.Cleanup(func() { + InvalidateVolumeIdLocationCache(vid) + InvalidateVolumeIdLocationCache(otherVid) + }) + + chunks := []*ChunkInfo{ + {Fid: otherVid + ",01637037d6", Offset: 0, Size: int64(len(first))}, + {Fid: vid + ",02637037d6", Offset: int64(len(first)), Size: int64(len(second))}, + } + reader := NewChunkedFileReader(chunks, master, dial) + defer reader.Close() + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read: %v", err) + } + if want := append(append([]byte(nil), first...), second...); !bytes.Equal(got, want) { + t.Fatalf("read %d bytes, want %d", len(got), len(want)) + } + if fake.lookups() != 1 { + t.Fatalf("master lookups = %d, want 1", fake.lookups()) + } +} + +func TestChunkedFileReaderTreatsAShortBodyAsFailed(t *testing.T) { + const vid = "91206" + data := testChunkData(2048) + empty := newChunkReplica(t, nil, -1) + healthy := newChunkReplica(t, data, -1) + vc.Set(vid, []Location{{Url: empty.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials())) + defer reader.Close() + got, err := io.ReadAll(reader) + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("read: %d bytes, err %v; want %d bytes", len(got), err, len(data)) + } + if len(empty.requests()) != 1 || len(healthy.requests()) != 1 { + t.Fatalf("requests: empty %d, healthy %d; want 1 and 1", len(empty.requests()), len(healthy.requests())) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("client gone") } + +func TestChunkedFileReaderStopsWhenTheClientGoesAway(t *testing.T) { + const vid = "91204" + data := testChunkData(1024) + a, b := newChunkReplica(t, data, -1), newChunkReplica(t, data, -1) + vc.Set(vid, []Location{{Url: a.host()}, {Url: b.host()}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials())) + if _, err := reader.WriteTo(failingWriter{}); err == nil || err.Error() != "client gone" { + t.Fatalf("WriteTo error = %v, want the writer's error", err) + } + if n := len(a.requests()) + len(b.requests()); n != 1 { + t.Fatalf("replica requests = %d, want 1", n) + } + if _, err := vc.Get(vid); err != nil { + t.Fatalf("a client write error dropped the cached locations: %v", err) + } +} diff --git a/weed/operation/lookup.go b/weed/operation/lookup.go index a82504277a..77326a10ac 100644 --- a/weed/operation/lookup.go +++ b/weed/operation/lookup.go @@ -51,31 +51,41 @@ var ( // authorization the master stamped on the volume; pass it through to the // volume server on the read request. func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrl string, jwt string, err error) { + fullUrls, jwt, err := lookupFileIdUrls(masterFn, grpcDialOption, fileId) + if err != nil { + return "", jwt, err + } + return fullUrls[0], jwt, nil +} + +// lookupFileIdUrls returns the read URLs of every replica of fileId, local +// replicas first, each group in random order. +func lookupFileIdUrls(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrls []string, jwt string, err error) { parts := strings.Split(fileId, ",") if len(parts) != 2 { - return "", jwt, errors.New("Invalid fileId " + fileId) + return nil, jwt, errors.New("Invalid fileId " + fileId) } lookup, lookupError := LookupVolumeId(masterFn, grpcDialOption, parts[0]) if lookupError != nil { - return "", jwt, lookupError + return nil, jwt, lookupError } if len(lookup.Locations) == 0 { - return "", jwt, errors.New("File Not Found") + return nil, jwt, errors.New("File Not Found") } - localUrls := make([]string, 0, len(lookup.Locations)) + var localUrls, remoteUrls []string for _, loc := range lookup.Locations { - if !loc.DataInRemote { - localUrls = append(localUrls, loc.Url) - } - } - if len(localUrls) == 0 { - for _, loc := range lookup.Locations { - localUrls = append(localUrls, loc.Url) + fullUrl := "http://" + loc.Url + "/" + fileId + if loc.DataInRemote { + remoteUrls = append(remoteUrls, fullUrl) + } else { + localUrls = append(localUrls, fullUrl) } } + rand.Shuffle(len(localUrls), func(i, j int) { localUrls[i], localUrls[j] = localUrls[j], localUrls[i] }) + rand.Shuffle(len(remoteUrls), func(i, j int) { remoteUrls[i], remoteUrls[j] = remoteUrls[j], remoteUrls[i] }) - return "http://" + localUrls[rand.IntN(len(localUrls))] + "/" + fileId, lookup.Jwt, nil + return append(localUrls, remoteUrls...), lookup.Jwt, nil } func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) { From 88b76b5890abfe9457dac6f7d8ff75c1e1194491 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:59:30 +0000 Subject: [PATCH 04/31] docs: regenerate star history chart --- note/star_history.svg | 2013 +++++++++++++++++++++-------------------- 1 file changed, 1008 insertions(+), 1005 deletions(-) diff --git a/note/star_history.svg b/note/star_history.svg index b7fb4891cc..c82cc2d852 100644 --- a/note/star_history.svg +++ b/note/star_history.svg @@ -6,7 +6,7 @@ - 2026-11-01T00:37:25.935647 + 2026-11-02T00:59:30.216249 image/svg+xml @@ -40,23 +40,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -301,18 +301,18 @@ L 352.912648 26.215385 - + - + - + @@ -325,18 +325,18 @@ L 448.648401 26.215385 - + - + - + @@ -349,18 +349,18 @@ L 544.253188 26.215385 - + - + - + @@ -484,41 +484,41 @@ z - + - - + - + - + - + - + - + - + - + @@ -577,18 +577,18 @@ L 709.2 268.125647 - + - + - + @@ -599,18 +599,18 @@ L 709.2 224.3653 - + - + - + @@ -621,18 +621,18 @@ L 709.2 180.604953 - + - + - + @@ -643,18 +643,18 @@ L 709.2 136.844606 - + - + - + - + - + - + @@ -813,934 +813,937 @@ z +" clip-path="url(#p09ad2cb364)" style="fill: none; stroke: #0969da; stroke-width: 1.6; stroke-linecap: square"/> + From 2ea63bad488107c4c760eb36b21a3425115fef72 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:33:07 -0800 Subject: [PATCH 05/31] filer.backup: keep local sink writes inside the sink directory (#12310) * localsink: skip keys that resolve outside the sink directory Co-Authored-By: Claude Opus 5.5 * filer: reject entry paths with dot segments Co-Authored-By: Claude Opus 5.5 * filer: test renames onto dot-segment targets Co-Authored-By: Claude Opus 5.5 * filer: keep the dot-segment rename test inside one bucket scope Co-Authored-By: Claude Opus 5.5 * localsink: resolve keys through an os.Root on the sink directory The lexical check alone lets a symlinked directory that already exists under the sink directory carry writes and deletes outside it. Create, rename and delete now go through an os.Root opened on the sink directory, so a symlink can only resolve inside it. Co-Authored-By: Claude Opus 5.5 * localsink: follow absolute symlinks that resolve inside the sink directory os.Root rejects every absolute symlink target, so a backup through an absolute link that points back into the sink failed, and a delete through it left the file behind. Resolve the parent of the key first and use the resolved path when it stays under the sink directory. Operations still go through os.Root, so a link swapped after the check cannot escape. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/command/filer_sync_process_test.go | 35 +++++ weed/filer/filer.go | 7 + weed/replication/sink/localsink/local_sink.go | 79 ++++++++-- .../sink/localsink/local_sink_dir_test.go | 146 ++++++++++++++++++ .../filer_grpc_server_create_route_test.go | 37 +++++ weed/server/filer_grpc_server_rename_test.go | 29 ++++ weed/util/fullpath.go | 9 ++ 7 files changed, 326 insertions(+), 16 deletions(-) create mode 100644 weed/replication/sink/localsink/local_sink_dir_test.go diff --git a/weed/command/filer_sync_process_test.go b/weed/command/filer_sync_process_test.go index 132902134a..3e81e0d914 100644 --- a/weed/command/filer_sync_process_test.go +++ b/weed/command/filer_sync_process_test.go @@ -1,11 +1,14 @@ package command import ( + "os" + "path/filepath" "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/replication/sink" + "github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink" "github.com/seaweedfs/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -343,3 +346,35 @@ func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) { t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys) } } + +func TestGenProcessFunctionLocalSinkStaysInsideTarget(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "backup") + dataSink := &localsink.LocalSink{Dir: target} + dataSink.SetSourceFiler(&source.FilerSource{}) + processFn := genProcessFunction("/", target, nil, nil, nil, nil, dataSink, true, false) + + for _, dir := range []string{"/safe", "/../escape"} { + err := processFn(&filer_pb.SubscribeMetadataResponse{ + Directory: dir, + EventNotification: &filer_pb.EventNotification{ + NewParentPath: dir, + NewEntry: &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + }, + }, + }) + if err != nil { + t.Fatalf("processFn %s: %v", dir, err) + } + } + + if _, err := os.Stat(filepath.Join(target, "safe", "marker.txt")); err != nil { + t.Fatalf("inside file missing: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) { + t.Fatalf("local sink wrote outside %s: %v", target, err) + } +} diff --git a/weed/filer/filer.go b/weed/filer/filer.go index bce66f5889..bae29d10bf 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -501,6 +501,10 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, return filer_pb.ErrEntryNameTooLong } + if entry.FullPath.HasDotSegment() { + return fmt.Errorf("invalid path %s", entry.FullPath) + } + if entry.IsDirectory() { entry.Attr.TtlSec = 0 } @@ -794,6 +798,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, } func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) { + if entry.FullPath.HasDotSegment() { + return fmt.Errorf("invalid path %s", entry.FullPath) + } ctx = f.syncWriteContext(ctx, entry.FullPath) if oldEntry != nil { entry.Attr.Crtime = oldEntry.Attr.Crtime diff --git a/weed/replication/sink/localsink/local_sink.go b/weed/replication/sink/localsink/local_sink.go index 8debf7993a..b66e8abff4 100644 --- a/weed/replication/sink/localsink/local_sink.go +++ b/weed/replication/sink/localsink/local_sink.go @@ -3,6 +3,7 @@ package localsink import ( "context" "fmt" + "math/rand/v2" "os" "path/filepath" "strings" @@ -39,6 +40,34 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool { return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/") } +func (localsink *LocalSink) isInsideDir(key string) (string, bool) { + rel, err := filepath.Rel(filepath.Clean(localsink.Dir), filepath.Clean(key)) + if err == nil && filepath.IsLocal(rel) { + return rel, true + } + glog.Warningf("local sink: skip %s outside %s", key, localsink.Dir) + return "", false +} + +// resolveInsideDir resolves symlinks in the parent of rel while it stays inside the sink directory. +func (localsink *LocalSink) resolveInsideDir(rel string) string { + base, err := filepath.EvalSymlinks(localsink.Dir) + if err != nil { + return rel + } + dir, rest := filepath.Dir(rel), filepath.Base(rel) + for dir != "." { + if resolved, err := filepath.EvalSymlinks(filepath.Join(base, dir)); err == nil { + if inside, err := filepath.Rel(base, resolved); err == nil && filepath.IsLocal(inside) { + return filepath.Join(inside, rest) + } + return rel + } + dir, rest = filepath.Dir(dir), filepath.Join(filepath.Base(dir), rest) + } + return rel +} + func (localsink *LocalSink) initialize(dir string, isIncremental bool) error { localsink.Dir = dir localsink.isIncremental = isIncremental @@ -69,8 +98,17 @@ func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeCh if localsink.isMultiPartEntry(key) { return nil } + rel, ok := localsink.isInsideDir(key) + if !ok { + return nil + } glog.V(4).Infof("Delete Entry key: %s", key) - if err := os.Remove(key); err != nil { + root, err := os.OpenRoot(localsink.Dir) + if err == nil { + err = root.Remove(localsink.resolveInsideDir(rel)) + root.Close() + } + if err != nil { glog.V(0).Infof("remove entry key %s: %s", key, err) } return nil @@ -81,6 +119,10 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa if entry.IsDirectory || localsink.isMultiPartEntry(key) { return nil } + rel, ok := localsink.isInsideDir(key) + if !ok { + return nil + } glog.V(4).Infof("Create Entry key: %s", key) totalSize := filer.FileSize(entry) @@ -89,35 +131,37 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa return fmt.Errorf("resolve chunks of %s: %w", key, err) } - dir := filepath.Dir(key) - - if _, err := os.Stat(dir); os.IsNotExist(err) { - glog.V(4).Infof("Create Directory key: %s", dir) - if err = os.MkdirAll(dir, 0755); err != nil { - return err - } + if err := os.MkdirAll(localsink.Dir, 0755); err != nil { + return err } + root, err := os.OpenRoot(localsink.Dir) + if err != nil { + return err + } + defer root.Close() - if entry.IsDirectory { - return os.Mkdir(key, os.FileMode(entry.Attributes.FileMode)) + rel = localsink.resolveInsideDir(rel) + dir := filepath.Dir(rel) + if err := root.MkdirAll(dir, 0755); err != nil { + return err } mode := os.FileMode(entry.Attributes.FileMode) - shortFileName := util.ToShortFileName(key) + shortFileName := util.ToShortFileName(rel) // Write to a temp file in the same directory, then atomically rename // on success. This prevents leaving a truncated/empty file if // decryption or chunk copy fails. - tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*") + tmpName := filepath.Join(dir, fmt.Sprintf(".seaweedfs-tmp-%d", rand.Uint64())) + tmpFile, err := root.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } - tmpName := tmpFile.Name() defer func() { // Clean up temp file on any error (rename removes it on success) if tmpFile != nil { tmpFile.Close() - os.Remove(tmpName) + root.Remove(tmpName) } }() @@ -151,8 +195,8 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa tmpFile = nil // prevent deferred cleanup // Atomic rename into final destination - if err := os.Rename(tmpName, shortFileName); err != nil { - os.Remove(tmpName) + if err := root.Rename(tmpName, shortFileName); err != nil { + root.Remove(tmpName) return err } @@ -164,6 +208,9 @@ func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, ne if localsink.isMultiPartEntry(key) { return true, nil } + if _, ok := localsink.isInsideDir(key); !ok { + return true, nil + } glog.V(4).Infof("Update Entry key: %s", key) // do delete and create foundExistingEntry = util.FileExists(key) diff --git a/weed/replication/sink/localsink/local_sink_dir_test.go b/weed/replication/sink/localsink/local_sink_dir_test.go new file mode 100644 index 0000000000..c044ca500a --- /dev/null +++ b/weed/replication/sink/localsink/local_sink_dir_test.go @@ -0,0 +1,146 @@ +package localsink + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/replication/source" +) + +func TestLocalSinkStaysInsideDir(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "backup") + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := func() *filer_pb.Entry { + return &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + } + + inside := filepath.Join(dir, "safe", "marker.txt") + if err := sink.CreateEntry(inside, entry(), nil); err != nil { + t.Fatalf("create inside: %v", err) + } + if _, err := os.Stat(inside); err != nil { + t.Fatalf("inside file missing: %v", err) + } + + outside := filepath.Join(dir, "..", "escape", "marker.txt") + if err := sink.CreateEntry(outside, entry(), nil); err != nil { + t.Fatalf("create outside: %v", err) + } + if _, err := sink.UpdateEntry(outside, nil, "", entry(), false, nil); err != nil { + t.Fatalf("update outside: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) { + t.Fatalf("sink wrote outside %s: %v", dir, err) + } + + victim := filepath.Join(root, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "..", "victim.txt"), false, false, nil); err != nil { + t.Fatalf("delete outside: %v", err) + } + if _, err := os.Stat(victim); err != nil { + t.Fatalf("sink deleted a file outside %s: %v", dir, err) + } +} + +func TestLocalSinkDoesNotFollowSymlinksOutOfDir(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "backup") + outside := filepath.Join(root, "outside") + for _, d := range []string{dir, outside} { + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(outside, filepath.Join(dir, "link")); err != nil { + t.Skipf("symlink: %v", err) + } + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + if err := sink.CreateEntry(filepath.Join(dir, "link", "marker.txt"), entry, nil); err == nil { + t.Fatalf("create through a symlink out of %s succeeded", dir) + } + if _, err := os.Stat(filepath.Join(outside, "marker.txt")); !os.IsNotExist(err) { + t.Fatalf("sink wrote through a symlink out of %s: %v", dir, err) + } + + victim := filepath.Join(outside, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "link", "victim.txt"), false, false, nil); err != nil { + t.Fatalf("delete through link: %v", err) + } + if _, err := os.Stat(victim); err != nil { + t.Fatalf("sink deleted through a symlink out of %s: %v", dir, err) + } + + if err := os.Mkdir(filepath.Join(dir, "real"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real", filepath.Join(dir, "inner")); err != nil { + t.Fatal(err) + } + if err := sink.CreateEntry(filepath.Join(dir, "inner", "marker.txt"), entry, nil); err != nil { + t.Fatalf("create through a symlink inside %s: %v", dir, err) + } + if _, err := os.Stat(filepath.Join(dir, "real", "marker.txt")); err != nil { + t.Fatalf("inside symlink target missing: %v", err) + } +} + +func TestLocalSinkFollowsAbsoluteSymlinksInsideDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "backup") + target := filepath.Join(dir, "real") + if err := os.MkdirAll(target, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(dir, "abs")); err != nil { + t.Skipf("symlink: %v", err) + } + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + for _, name := range []string{"marker.txt", filepath.Join("sub", "marker.txt")} { + if err := sink.CreateEntry(filepath.Join(dir, "abs", name), entry, nil); err != nil { + t.Fatalf("create %s through an absolute symlink inside %s: %v", name, dir, err) + } + if _, err := os.Stat(filepath.Join(target, name)); err != nil { + t.Fatalf("absolute symlink target %s missing: %v", name, err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "abs", name), false, false, nil); err != nil { + t.Fatalf("delete %s through an absolute symlink: %v", name, err) + } + if _, err := os.Stat(filepath.Join(target, name)); !os.IsNotExist(err) { + t.Fatalf("delete through an absolute symlink left %s: %v", name, err) + } + } + if _, err := os.Lstat(filepath.Join(dir, "abs")); err != nil { + t.Fatalf("symlink itself removed: %v", err) + } +} diff --git a/weed/server/filer_grpc_server_create_route_test.go b/weed/server/filer_grpc_server_create_route_test.go index 0c266c5397..8bb4bbbc33 100644 --- a/weed/server/filer_grpc_server_create_route_test.go +++ b/weed/server/filer_grpc_server_create_route_test.go @@ -232,3 +232,40 @@ func TestCreateEntryFailsWhenPriorOwnerIsUnreachable(t *testing.T) { t.Fatal("a create routed away must not also write locally") } } + +func TestCreateEntryRejectsDotSegments(t *testing.T) { + fs, store := createRouteServer(t) + + for _, tc := range []struct{ dir, name string }{ + {"/../escape", "marker.txt"}, + {"/test/./sub", "marker.txt"}, + {"/test", ".."}, + } { + req := createReq(tc.name, false) + req.Directory = tc.dir + resp, err := fs.CreateEntry(context.Background(), req) + if err == nil && resp.Error == "" { + t.Fatalf("CreateEntry %s/%s must be rejected", tc.dir, tc.name) + } + if _, found := store.entries[string(util.NewFullPath(tc.dir, tc.name))]; found { + t.Fatalf("CreateEntry %s/%s stored the raw path", tc.dir, tc.name) + } + } + + resp, err := fs.CreateEntry(context.Background(), createReq("marker.txt", false)) + if err != nil || resp.Error != "" { + t.Fatalf("CreateEntry /test/marker.txt: err=%v resp=%v", err, resp) + } + + update := &filer_pb.UpdateEntryRequest{ + Directory: "/x/../test", + Entry: createReq("marker.txt", false).Entry, + } + update.Entry.Attributes.Mtime++ + if _, err := fs.UpdateEntry(context.Background(), update); err == nil { + t.Fatal("UpdateEntry with a dot segment must be rejected") + } + if _, found := store.entries["/x/../test/marker.txt"]; found { + t.Fatal("UpdateEntry stored the raw path") + } +} diff --git a/weed/server/filer_grpc_server_rename_test.go b/weed/server/filer_grpc_server_rename_test.go index 10d6941406..b793025472 100644 --- a/weed/server/filer_grpc_server_rename_test.go +++ b/weed/server/filer_grpc_server_rename_test.go @@ -559,3 +559,32 @@ func TestAtomicRenameReleasesQueuedStampsOnCommitFailure(t *testing.T) { t.Fatalf("LocalDeliveredThroughTsNs = %d, want %d: a leaked stamp still caps the claim", got, now) } } + +func TestAtomicRenameEntryRejectsDotSegmentTarget(t *testing.T) { + for _, tc := range []struct{ dir, name string }{ + {"/../escape", "dst.txt"}, + {"/", ".."}, + } { + store := newRenameTestStore() + store.entries["/src.txt"] = newFileEntry("/src.txt", 101) + f := newRenameTestFiler(t, store) + f.DirBucketsPath = "/buckets" + server := &FilerServer{filer: f, entryLockTable: util.NewLockTable[util.FullPath]()} + + _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ + OldDirectory: "/", + OldName: "src.txt", + NewDirectory: tc.dir, + NewName: tc.name, + }) + if err == nil { + t.Fatalf("rename to %s/%s must be rejected", tc.dir, tc.name) + } + if _, found := store.entries["/src.txt"]; !found { + t.Fatalf("rename to %s/%s removed the source", tc.dir, tc.name) + } + if len(store.entries) != 1 { + t.Fatalf("rename to %s/%s left entries %v", tc.dir, tc.name, store.entries) + } + } +} diff --git a/weed/util/fullpath.go b/weed/util/fullpath.go index 866592f95d..529344a548 100644 --- a/weed/util/fullpath.go +++ b/weed/util/fullpath.go @@ -76,6 +76,15 @@ func (fp FullPath) IsLongerFileName(maxFilenameLength uint32) bool { return uint32(len([]byte(fp.Name()))) > maxFilenameLength } +func (fp FullPath) HasDotSegment() bool { + for _, seg := range strings.Split(string(fp), "/") { + if seg == "." || seg == ".." { + return true + } + } + return false +} + func (fp FullPath) Child(name string) FullPath { dir := string(fp) noPrefix := name From 965c6c06fb12db7880de1d02a18d586504627c3d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:34:04 -0800 Subject: [PATCH 06/31] Ship weed-vector in release assets and Docker images (#12322) * ci(release): publish weed-vector binaries Tagged releases now carry weed-vector_linux_{amd64,arm64}.tar.gz, static musl builds made the same way as weed-worker's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * docker: carry weed-vector in the image The image takes a pre-built weed-vector the way it takes weed-worker, or an empty placeholder where none was built. `vector` runs it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * ci(docker): build weed-vector for the images The container workflows cross-compile weed-vector for amd64 and arm64 beside weed-worker and place it in the Docker context. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * install.sh: add the vector component `--component vector` installs weed-vector from a release, and `all` adds it when the release carries one, as it does for the worker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * ci(docker): skip weed-vector for source refs that predate it The manual latest-image rebuild accepts any source_ref; one without the engine's server crate now builds its image without weed-vector instead of failing the Rust job. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * docker: have the vector engine listen beyond loopback weed-vector listens on 127.0.0.1 by default, which a gateway in another container cannot reach. `vector` now adds --listen 0.0.0.0:8090 unless a listen address is given by flag or WEED_VECTOR_LISTEN. The engine registers its container address for a wildcard listen. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/container_dev.yml | 20 ++++- .github/workflows/container_latest.yml | 33 +++++++- .../workflows/container_release_unified.yml | 25 +++++- .github/workflows/rust_binaries_release.yml | 77 +++++++++++++++++++ docker/Dockerfile.go_build | 13 +++- docker/entrypoint.sh | 13 ++++ docker/weed-vector-prebuilt/.gitkeep | 0 install.sh | 54 ++++++++++++- 8 files changed, 226 insertions(+), 9 deletions(-) create mode 100644 docker/weed-vector-prebuilt/.gitkeep diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 45619a21e0..4336a3ce49 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -8,6 +8,7 @@ on: - 'seaweed-volume/**' - 'seaweed-common/**' - 'seaweed-worker/**' + - 'seaweed-vector/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -50,7 +51,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -83,6 +85,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -90,6 +98,7 @@ jobs: path: | weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -127,8 +136,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 668c6046f6..f5a445e07a 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -96,7 +96,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -137,6 +138,17 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + # An older source_ref predates the engine; its image goes without one. + if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then + echo "No vector engine in this source ref" + exit 0 + fi + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -145,6 +157,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -217,8 +230,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta @@ -342,8 +364,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index c78e5da7dc..47f040fd83 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -43,9 +43,9 @@ concurrency: jobs: # ── Pre-build the Rust binaries natively ──────────────────────────── - # The volume server and the Rust maintenance worker, cross-compiled for - # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into - # ~15 minutes of native compilation. + # The volume server, the Rust maintenance worker and the vector engine, + # cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated + # cargo build into ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-24.04 strategy: @@ -73,7 +73,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -114,6 +115,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -122,6 +129,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -198,8 +206,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 3999c6df34..f6c6359ac5 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -188,6 +188,83 @@ jobs: weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + # The vector engine behind the S3 Vectors API: Linux only, like the worker, + # whose lance dependency tree it shares. + build-rust-vector-linux: + permissions: + contents: write + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-musl + asset_suffix: linux_arm64 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the musl target + uses: ./.github/actions/rust-musl-target + with: + target: ${{ matrix.target }} + + - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-vector/target/${{ matrix.target }}/release + key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }} + restore-keys: | + rust-vector-release-${{ matrix.target }}- + + - name: Use the vendored protoc + run: | + cd seaweed-vector + cargo fetch + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector + tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector + rm weed-vector + md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-vector-${{ matrix.asset_suffix }} + path: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + build-rust-volume-darwin: permissions: contents: write diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index a2eb20568d..22e4674290 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -33,6 +33,7 @@ ARG TARGETARCH ARG TAGS COPY weed-volume-prebuilt/ /prebuilt/ COPY weed-worker-prebuilt/ /prebuilt-worker/ +COPY weed-vector-prebuilt/ /prebuilt-vector/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume # seaweed-common is a path dependency of seaweed-volume that lives beside it, # so the source build below needs it in the same relative position. @@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ echo "No pre-built Rust worker for ${TARGETARCH}" && \ touch /weed-worker; \ fi +# The vector engine carries the same lance tree, so it is pre-built or absent too. +RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \ + echo "Using pre-built vector engine for ${TARGETARCH}" && \ + cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \ + else \ + echo "No pre-built vector engine for ${TARGETARCH}" && \ + touch /weed-vector; \ + fi # Pre-built binaries arrive via GitHub Actions artifacts, which drop the # executable bit, so the copied file is 0644 and exec fails with "Permission # denied". Restore it (no-op for the empty placeholders, which stay size 0). -RUN chmod 0755 /weed-volume /weed-worker +RUN chmod 0755 /weed-volume /weed-worker /weed-vector FROM alpine AS final LABEL author="Chris Lu" @@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=rust_builder /weed-volume /usr/bin/weed-volume # Same for the Rust maintenance worker, which serves Lance table buckets COPY --from=rust_builder /weed-worker /usr/bin/weed-worker +# And the vector engine behind the S3 Vectors API +COPY --from=rust_builder /weed-vector /usr/bin/weed-vector RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0bbd507871..8e6ee840c5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -100,6 +100,19 @@ case "$1" in exec /usr/bin/weed-worker "$@" ;; + 'vector') + shift + if [ ! -s /usr/bin/weed-vector ]; then + echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2 + exit 1 + fi + # The gateway dials the engine from another container, so not loopback. + if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then + set -- --listen 0.0.0.0:8090 "$@" + fi + exec /usr/bin/weed-vector "$@" + ;; + 'server') ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" if isArgPassed "-volume.max" "$@"; then diff --git a/docker/weed-vector-prebuilt/.gitkeep b/docker/weed-vector-prebuilt/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/install.sh b/install.sh index 8bede247af..72574bf4cc 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,7 @@ # curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin # # Options: -# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed) +# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed) # --version VER Release version tag (default: latest) # --large-disk Use large disk variant (5-byte offset, 8TB max volume) # --dir DIR Installation directory (default: /usr/local/bin) @@ -23,6 +23,7 @@ VERSION="" LARGE_DISK=false INSTALL_DIR="/usr/local/bin" WORKER_INSTALLED=false +VECTOR_INSTALLED=false # Colors (if terminal supports them) if [ -t 1 ]; then @@ -156,6 +157,12 @@ worker_asset_name() { echo "weed-worker_${os}_${arch}.tar.gz" } +# The vector engine behind the S3 Vectors API, built like the worker. +vector_asset_name() { + local os="$1" arch="$2" + echo "weed-vector_${os}_${arch}.tar.gz" +} + # Install a single component install_component() { local component="$1" os="$2" arch="$3" @@ -264,8 +271,38 @@ install_component() { ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker" ;; + vector) + # Published for linux amd64/arm64 only, like the worker. + case "$os" in + linux) ;; + *) error "The vector engine is not available for ${os}. Supported: linux" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(vector_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + local vector_bin + vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)" + if [ -z "$vector_bin" ]; then + error "Could not find weed-vector binary in archive" + fi + + chmod +x "$vector_bin" + install_binary "$vector_bin" "weed-vector" + VECTOR_INSTALLED=true + ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector" + ;; + *) - error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all" + error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all" ;; esac @@ -327,6 +364,13 @@ main() { else install_component "worker-rust" "$os" "$arch" fi + if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then + warn "Skipping the vector engine: no build for ${os}/${arch}" + elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then + warn "Skipping the vector engine: ${VERSION} does not carry one" + else + install_component "vector" "$os" "$arch" + fi ;; *) install_component "$COMPONENT" "$os" "$arch" @@ -344,6 +388,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker: ${INSTALL_DIR}/weed-worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector: ${INSTALL_DIR}/weed-vector" + fi echo "" info "Quick start:" info " weed master # Start master server" @@ -352,6 +399,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector --help # The vector engine for S3 vector buckets" + fi } main From d84ae4f2ab52d3f8f7bf8913a96c345d3d081230 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:35:37 -0800 Subject: [PATCH 07/31] volume: don't let an unreachable tier backend hold up volume server startup (#12312) * s3 tier backend: bound connect and response-header waits The tier client used the SDK default transport: a 30s dial timeout and no limit on waiting for response headers. A blackholed endpoint cost about two minutes per request with the SDK's retries, and an endpoint that accepts connections but never answers held the request forever. Dial with a 10s timeout and give up on a response whose headers do not arrive within a minute. Co-Authored-By: Claude Opus 5.5 * volume: stop the disk scan from waiting on a stalled tier backend A volume server registers with the master only after every volume on its disks has loaded, and loading a tiered volume reads its superblock from the remote object. With the S3 endpoint unreachable, each read waited out the client's timeouts and retries on one of the loader workers, so start-up grew with the number of tiered volumes and the local volumes were not served until it finished. A tiered volume already loads without its superblock when the read fails. Give that read 10s, and once it stalls, let the backend's other volumes skip it for a minute instead of each waiting out the timeout. The volume reads from the backend as before once the endpoint answers again. Co-Authored-By: Claude Opus 5.5 * seaweed-volume: bound connect and read waits of the S3 tier client Mirror the Go tier client: a 10s connect timeout instead of the SDK's 3.1s default, and a minute to receive a response, so an endpoint that accepts connections but never answers cannot hold a read forever. Co-Authored-By: Claude Opus 5.5 * seaweed-volume: stop volume loading from waiting on a stalled tier backend Mirror the Go loader: the superblock read of a tiered volume gets 10s, and once it stalls the backend's other volumes skip it for a minute. The volume loads with its .vif version as before when the read fails, and reads from the backend once the endpoint answers again. Co-Authored-By: Claude Opus 5.5 * seaweed-volume: keep the .vif replication when a remote superblock read fails Go applies the .vif replication to a tiered volume whether or not its superblock read succeeded, so a volume loaded while its backend is down still reports its replica placement. The Rust loader applied it only on success and reported the default placement instead. Co-Authored-By: Claude Opus 5.5 * volume: skip the .idx head repair while the tier backend is unreachable The repair reads needle headers from the remote .dat, so after the superblock read failed it would wait on the same backend again and hold up the load. It runs on the next load that reaches the backend. Co-Authored-By: Claude Opus 5.5 * seaweed-volume: skip the .idx head repair while the tier backend is unreachable Mirrors the Go load: after the remote superblock read failed, the repair's remote header reads would wait on the same backend again. Co-Authored-By: Claude Opus 5.5 * s3 tier backend: give up a ranged read whose body stops arriving The response-header timeout ends once headers arrive, so an endpoint that answers and then sends no body held ReadAt, and a superblock read's worker, forever. Cancel the request once the body makes no progress for s3ReadTimeout. Co-Authored-By: Claude Opus 5.5 * seaweed-volume: test that a tier read whose body stalls gives up The SDK's stalled-stream protection already fails it, matching the Go read's no-progress cancel. Co-Authored-By: Claude Opus 5.5 * volume(rust): keep the S3 read timeout off upload_part The SDK's read timeout runs from the start of the request, so a 64 MiB part on a slow link would time out while its body is still being sent. Go's ResponseHeaderTimeout only starts after the body is written. upload_part now keeps the connect timeout and drops the read timeout; stalled-stream protection still fails an upload that stops moving. Co-Authored-By: Claude Opus 5.5 * volume(rust): bound the upload_part response wait from when the part is sent Without a read timeout, an upload_part whose body was accepted but never answered waited forever. The part body now goes to the connection in 64 KiB chunks, and once the last chunk is taken the response must arrive within S3_READ_TIMEOUT, like Go's ResponseHeaderTimeout. A retry rebuilds the body and restarts that clock. The body keeps its in-memory contents, so signing and the checksum header are unchanged. Co-Authored-By: Claude Opus 5.5 * volume(rust): keep a retried part's response clock from an earlier attempt The send state was a single flag, and a retry body could be rebuilt and fully sent before the waiter saw the flag drop, so the waiter kept the earlier attempt's deadline. Each body now carries its attempt number, only the latest attempt marks the part sent, and the deadline holds only while no later attempt has started. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/remote_storage/s3_tier.rs | 431 +++++++++++++++++- seaweed-volume/src/storage/volume.rs | 404 ++++++++++++++-- .../src/storage/volume_idx_repair.rs | 4 +- weed/storage/backend/s3_backend/s3_backend.go | 11 +- .../backend/s3_backend/s3_backend_test.go | 4 +- .../storage/backend/s3_backend/s3_sessions.go | 21 +- .../backend/s3_backend/s3_sessions_test.go | 101 +++- weed/storage/volume_loading.go | 6 +- .../volume_remote_superblock_stall_test.go | 185 ++++++++ weed/storage/volume_super_block.go | 43 +- 10 files changed, 1143 insertions(+), 67 deletions(-) create mode 100644 weed/storage/volume_remote_superblock_stall_test.go diff --git a/seaweed-volume/src/remote_storage/s3_tier.rs b/seaweed-volume/src/remote_storage/s3_tier.rs index 1bcea50bb7..98603fb25f 100644 --- a/seaweed-volume/src/remote_storage/s3_tier.rs +++ b/seaweed-volume/src/remote_storage/s3_tier.rs @@ -5,21 +5,153 @@ use std::collections::HashMap; use std::future::Future; +use std::pin::Pin; use std::sync::{Arc, OnceLock, RwLock}; +use std::task::{Context, Poll}; +use std::time::Duration; use aws_sdk_s3::Client; use aws_sdk_s3::config::http::HttpResponse; +use aws_sdk_s3::config::timeout::TimeoutConfig; use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region}; use aws_sdk_s3::error::{DisplayErrorContext, SdkError}; use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; +use aws_sdk_s3::operation::upload_part::UploadPartOutput; +use aws_sdk_s3::operation::upload_part::builders::UploadPartFluentBuilder; +use aws_sdk_s3::primitives::{ByteStream, SdkBody}; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use bytes::Bytes; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use tokio::sync::Semaphore; +use tokio::sync::{Semaphore, watch}; /// Concurrency limit for multipart upload/download (matches Go's s3manager). const CONCURRENCY: usize = 5; +/// A tier endpoint that drops packets or accepts connections without answering +/// must not hold a read, or a volume server's startup, for minutes per request. +/// Matches Go's s3ConnectTimeout and s3ReadTimeout. The SDK's stalled-stream +/// protection already fails a body that stops arriving. +const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const S3_READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// The SDK's read timeout runs from the start of the request, so on +/// upload_part it would also cap sending the part body. send_part bounds the +/// response wait instead, from when the body is written, like Go's +/// ResponseHeaderTimeout. +fn upload_part_config() -> aws_sdk_s3::config::Builder { + aws_sdk_s3::config::Builder::default().timeout_config( + TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .disable_read_timeout() + .build(), + ) +} + +/// A part goes to the connection in chunks this size, so its last chunk is +/// taken only once nearly all of the part has been written. +const UPLOAD_CHUNK_SIZE: usize = 64 * 1024; + +/// The latest attempt to send a part, and whether its body has been sent. +type PartSent = Arc>; + +struct PartBody { + rest: Bytes, + attempt: u64, + sent: PartSent, +} + +impl http_body::Body for PartBody { + type Data = Bytes; + type Error = std::convert::Infallible; + + fn poll_frame( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + if self.rest.is_empty() { + return Poll::Ready(None); + } + let n = self.rest.len().min(UPLOAD_CHUNK_SIZE); + let chunk = self.rest.split_to(n); + if self.rest.is_empty() { + let attempt = self.attempt; + self.sent.send_if_modified(|state| { + let latest = state.0 == attempt; + state.1 |= latest; + latest + }); + } + Poll::Ready(Some(Ok(http_body::Frame::data(chunk)))) + } + + fn is_end_stream(&self) -> bool { + self.rest.is_empty() + } + + fn size_hint(&self) -> http_body::SizeHint { + http_body::SizeHint::with_exact(self.rest.len() as u64) + } +} + +/// Sends one multipart part, giving up once `response_timeout` passes after +/// the whole part was handed to the connection without a response. A retry +/// rebuilds the body and restarts that clock. The body keeps its in-memory +/// contents, so signing and the checksum header are unchanged. +async fn send_part( + request: UploadPartFluentBuilder, + data: Bytes, + response_timeout: Duration, +) -> Result { + let sent: PartSent = Arc::new(watch::channel((0, false)).0); + let body = { + let sent = sent.clone(); + SdkBody::from(data.clone()).map_preserve_contents(move |_| { + let mut attempt = 0; + sent.send_modify(|state| { + *state = (state.0 + 1, data.is_empty()); + attempt = state.0; + }); + SdkBody::from_body_1_x(PartBody { + rest: data.clone(), + attempt, + sent: sent.clone(), + }) + }) + }; + tokio::select! { + resp = request + .body(ByteStream::new(body)) + .customize() + .config_override(upload_part_config()) + .send() => resp.map_err(|e| DisplayErrorContext(&e).to_string()), + () = wait_unanswered(sent, response_timeout) => Err(format!( + "no response within {response_timeout:?} after sending the part" + )), + } +} + +/// Completes once an attempt's body has been sent and `response_timeout` +/// passes before a later attempt starts. +async fn wait_unanswered(sent: PartSent, response_timeout: Duration) { + let mut state = sent.subscribe(); + loop { + let attempt = state + .wait_for(|&(_, done)| done) + .await + .map_or(0, |state| state.0); + if tokio::time::timeout( + response_timeout, + state.wait_for(|&(latest, _)| latest != attempt), + ) + .await + .is_err() + { + return; + } + } +} + /// A tier transfer failure. The variant is what callers match on; the /// message is the operator-facing text. #[derive(Debug, thiserror::Error)] @@ -107,7 +239,13 @@ impl S3TierBackend { .behavior_version(BehaviorVersion::latest()) .region(Region::new(region.to_string())) .credentials_provider(credentials) - .force_path_style(config.force_path_style); + .force_path_style(config.force_path_style) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_READ_TIMEOUT) + .build(), + ); if !config.endpoint.is_empty() { s3_config = s3_config.endpoint_url(&config.endpoint); @@ -231,23 +369,23 @@ impl S3TierBackend { TierError::Io(format!("failed to read file at offset {}: {}", off, e)) })?; - let upload_part_resp = client - .upload_part() - .bucket(&bucket) - .key(&key) - .upload_id(&upload_id) - .part_number(pn) - .body(buf.into()) - .send() - .await - .map_err(|e| { - TierError::Io(format!( - "failed to upload part {} at offset {}: {}", - pn, - off, - DisplayErrorContext(&e) - )) - })?; + let upload_part_resp = send_part( + client + .upload_part() + .bucket(&bucket) + .key(&key) + .upload_id(&upload_id) + .part_number(pn), + buf.into(), + S3_READ_TIMEOUT, + ) + .await + .map_err(|e| { + TierError::Io(format!( + "failed to upload part {} at offset {}: {}", + pn, off, e + )) + })?; let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string(); @@ -560,6 +698,25 @@ impl S3TierBackend { }) } + /// `read_range_blocking` that gives up once `timeout` passes, returning + /// `Ok(None)`. + pub fn read_range_blocking_within( + self: &Arc, + key: &str, + offset: u64, + size: usize, + timeout: Duration, + ) -> Result>, TierError> { + let backend = Arc::clone(self); + let key = key.to_string(); + block_on_tier_future(async move { + match tokio::time::timeout(timeout, backend.read_range(&key, offset, size)).await { + Ok(read) => read.map(Some), + Err(_) => Ok(None), + } + }) + } + pub fn read_range_blocking( &self, key: &str, @@ -725,6 +882,13 @@ fn describe_join_error(join_error: tokio::task::JoinError) -> TierError { mod tests { use super::*; use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY}; + use aws_sdk_s3::config::http::HttpRequest; + use aws_sdk_s3::config::{HttpClient, RuntimeComponents}; + use aws_smithy_runtime_api::client::http::{ + HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, + }; + use aws_smithy_runtime_api::http::StatusCode; + use http_body::Body as _; use std::collections::HashSet; use tokio::runtime::Handle; @@ -897,6 +1061,235 @@ mod tests { assert!(matches!(err, TierError::Io(_)), "{err:?}"); } + #[test] + fn backend_bounds_connect_and_read_waits() { + let (_listener, backend) = silent_backend(); + let timeouts = backend.client.config().timeout_config().unwrap(); + assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT)); + assert_eq!(timeouts.read_timeout(), Some(S3_READ_TIMEOUT)); + } + + #[derive(Debug)] + struct CaptureTimeouts(Arc>>); + + impl aws_sdk_s3::config::Intercept for CaptureTimeouts { + fn name(&self) -> &'static str { + "CaptureTimeouts" + } + + fn read_before_transmit( + &self, + _context: &aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &aws_sdk_s3::config::RuntimeComponents, + cfg: &mut aws_sdk_s3::config::ConfigBag, + ) -> Result<(), aws_sdk_s3::error::BoxError> { + *self.0.lock().unwrap() = cfg.load::().cloned(); + Err("captured".into()) + } + } + + #[tokio::test] + async fn upload_part_keeps_the_connect_timeout_without_the_read_timeout() { + let (_listener, backend) = silent_backend(); + let seen = Arc::new(std::sync::Mutex::new(None)); + let _ = backend + .client + .upload_part() + .bucket("bucket") + .key("key") + .upload_id("upload") + .part_number(1) + .body(vec![0u8; 8].into()) + .customize() + .config_override(upload_part_config().interceptor(CaptureTimeouts(seen.clone()))) + .send() + .await; + let timeouts = seen + .lock() + .unwrap() + .clone() + .expect("request never reached transmit"); + assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT)); + assert_eq!(timeouts.read_timeout(), None); + } + + fn upload_part_request(backend: &S3TierBackend) -> UploadPartFluentBuilder { + backend + .client + .upload_part() + .bucket("bucket") + .key("key") + .upload_id("upload") + .part_number(1) + } + + #[tokio::test] + async fn send_part_gives_up_when_no_response_follows_the_body() { + let (_listener, backend) = silent_backend(); + let sent = tokio::time::timeout( + Duration::from_secs(30), + send_part( + upload_part_request(&backend), + Bytes::from_static(&[0u8; 8]), + Duration::from_millis(200), + ), + ) + .await + .expect("a part with no response did not give up"); + let err = sent.unwrap_err(); + assert!(err.starts_with("no response within 200ms"), "{err}"); + } + + #[tokio::test(start_paused = true)] + async fn a_retry_restarts_the_response_clock() { + let sent: PartSent = Arc::new(watch::channel((1, true)).0); + let mut unanswered = + std::pin::pin!(wait_unanswered(sent.clone(), Duration::from_millis(200))); + let wait = |ms| tokio::time::sleep(Duration::from_millis(ms)); + tokio::select! { + () = unanswered.as_mut() => panic!("gave up early"), + () = wait(150) => {} + } + // The retry starts and finishes sending before the waiter runs again. + sent.send_replace((2, false)); + sent.send_replace((2, true)); + tokio::select! { + () = unanswered.as_mut() => panic!("kept the first attempt's clock"), + () = wait(150) => {} + } + tokio::select! { + () = unanswered => {} + () = wait(100) => panic!("the retry never timed out"), + } + } + + /// Takes the request body one chunk per `pause`, then answers at once. + #[derive(Debug, Clone)] + struct SlowReader { + pause: Duration, + } + + impl HttpConnector for SlowReader { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let pause = self.pause; + HttpConnectorFuture::new(async move { + let mut body = std::pin::pin!(request.into_body()); + loop { + tokio::time::sleep(pause).await; + let frame = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await; + if frame.is_none() || body.is_end_stream() { + break; + } + } + let mut response = + HttpResponse::new(StatusCode::try_from(200).unwrap(), SdkBody::empty()); + response.headers_mut().insert("ETag", "\"etag\""); + Ok(response) + }) + } + } + + impl HttpClient for SlowReader { + fn http_connector( + &self, + _settings: &HttpConnectorSettings, + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + SharedHttpConnector::new(self.clone()) + } + } + + #[tokio::test] + async fn send_part_does_not_cap_a_slow_body() { + let config = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test")) + .endpoint_url("http://127.0.0.1:1") + .force_path_style(true) + .http_client(SlowReader { + pause: Duration::from_millis(300), + }) + .build(); + let backend = S3TierBackend { + client: Client::from_conf(config), + bucket: "bucket".to_string(), + storage_class: "STANDARD".to_string(), + }; + let output = send_part( + upload_part_request(&backend), + Bytes::from(vec![0u8; 4 * UPLOAD_CHUNK_SIZE]), + Duration::from_millis(200), + ) + .await + .expect("a part still being sent was cut short"); + assert_eq!(output.e_tag(), Some("\"etag\"")); + } + + #[test] + fn read_range_blocking_within_gives_up_on_a_silent_endpoint() { + let (_listener, backend) = silent_backend(); + let start = std::time::Instant::now(); + let got = backend + .read_range_blocking_within("key", 0, 8, Duration::from_millis(200)) + .unwrap(); + assert!(got.is_none()); + assert!(start.elapsed() < Duration::from_secs(5)); + } + + #[tokio::test] + async fn read_range_gives_up_when_the_body_stalls() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let mut held = Vec::new(); + for mut conn in listener.incoming().flatten() { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + match std::io::Read::read(&mut conn, &mut byte) { + Ok(1) => request.push(byte[0]), + _ => break, + } + } + let _ = std::io::Write::write_all( + &mut conn, + b"HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n", + ); + held.push(conn); + } + }); + let backend = S3TierBackend::new(&S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket".to_string(), + endpoint, + storage_class: String::new(), + force_path_style: true, + }); + + let read = tokio::time::timeout(Duration::from_secs(30), backend.read_range("key", 0, 8)) + .await + .expect("a read whose body stalls did not give up"); + let err = read.unwrap_err(); + assert!(matches!(err, TierError::Io(_)), "{err:?}"); + } + + fn silent_backend() -> (std::net::TcpListener, Arc) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let backend = S3TierBackend::new(&S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket".to_string(), + endpoint: format!("http://{}", listener.local_addr().unwrap()), + storage_class: String::new(), + force_path_style: true, + }); + (listener, Arc::new(backend)) + } + #[test] fn read_range_blocking_no_such_key_is_not_found() { let err = backend_answering(404, NO_SUCH_KEY) diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 8fc3b7214a..b3824380b8 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -9,14 +9,14 @@ //! Matches Go's storage/volume.go, volume_loading.go, volume_read.go, //! volume_write.go, volume_super_block.go. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::ops::ControlFlow; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64, Ordering}; -use std::sync::{Arc, Condvar, Mutex, RwLock}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, RwLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; @@ -1198,6 +1198,33 @@ impl RemoteDatFile { .backend .read_range_blocking(&self.key, offset, buf.len()) .map_err(io::Error::other)?; + Self::fill(buf, offset, data) + } + + /// `read_exact_at` that fails with `TimedOut` once `timeout` passes. + pub(crate) fn read_exact_at_within( + &self, + buf: &mut [u8], + offset: u64, + timeout: Duration, + ) -> io::Result<()> { + match self + .backend + .read_range_blocking_within(&self.key, offset, buf.len(), timeout) + .map_err(io::Error::other)? + { + Some(data) => Self::fill(buf, offset, data), + None => Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "remote read at offset {} got no answer within {:?}", + offset, timeout + ), + )), + } + } + + fn fill(buf: &mut [u8], offset: u64, data: Vec) -> io::Result<()> { if data.len() != buf.len() { return Err(io::Error::new( io::ErrorKind::UnexpectedEof, @@ -1214,6 +1241,71 @@ impl RemoteDatFile { } } +/// A tiered volume loads without its superblock when the backend cannot serve +/// it; once a read stalls, the backend's other volumes skip it for a while. +/// Matches Go's remoteSuperBlockTimeout and remoteSuperBlockStallPeriod. +const REMOTE_SUPER_BLOCK_TIMEOUT: Duration = Duration::from_secs(10); +const REMOTE_SUPER_BLOCK_STALL_PERIOD: Duration = Duration::from_secs(60); +static REMOTE_SUPER_BLOCK_STALLS: LazyLock>> = + LazyLock::new(Default::default); + +fn read_super_block_bytes( + mut read: impl FnMut(&mut [u8], u64) -> Result<(), VolumeError>, +) -> Result, VolumeError> { + let mut header = [0u8; SUPER_BLOCK_SIZE]; + read(&mut header, 0)?; + + let extra_size = u16::from_be_bytes([header[6], header[7]]); + let mut full_buf = vec![0u8; SUPER_BLOCK_SIZE + extra_size as usize]; + full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header); + if extra_size > 0 { + read(&mut full_buf[SUPER_BLOCK_SIZE..], SUPER_BLOCK_SIZE as u64)?; + } + Ok(full_buf) +} + +fn read_remote_super_block( + storage_name: &str, + remote: &RemoteDatFile, + timeout: Duration, + stall_period: Duration, +) -> Result, VolumeError> { + if let Some(until) = REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().get(storage_name) + && Instant::now() < *until + { + return Err(VolumeError::Io(io::Error::other(format!( + "backend {} stalled on a super block read, skipped", + storage_name + )))); + } + let deadline = Instant::now() + timeout; + let result = read_super_block_bytes(|buf, offset| { + remote + .read_exact_at_within( + buf, + offset, + deadline.saturating_duration_since(Instant::now()), + ) + .map_err(VolumeError::from) + }); + match &result { + Ok(_) => { + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .remove(storage_name); + } + Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::TimedOut => { + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(storage_name.to_string(), Instant::now() + stall_period); + } + Err(_) => {} + } + result +} + // ============================================================================ // Volume // ============================================================================ @@ -1684,6 +1776,7 @@ impl Volume { ))); } + let mut remote_super_block_failed = false; if already_has_super_block { match self.read_super_block(io) { Ok(()) => { @@ -1700,6 +1793,12 @@ impl Volume { error = %e, "failed to read remote super block during load" ); + remote_super_block_failed = true; + let replication = self.volume_info.read().unwrap().replication.clone(); + if !replication.is_empty() { + self.super_block.write().unwrap().replica_placement = + ReplicaPlacement::from_string(&replication)?; + } } Err(e) => return Err(e), } @@ -1748,17 +1847,24 @@ impl Volume { // at the front of .idx. Best effort: a volume that cannot be // repaired is still servable for everything the surviving rows // index. - match self.repair_idx_head_tombstones(io) { - Ok(0) => {} - Ok(restored) => info!( + if remote_super_block_failed { + debug!( volume_id = self.id.0, - restored, "recovered overwritten .idx rows from .dat" - ), - Err(e) => warn!( - volume_id = self.id.0, - error = %e, - "recover overwritten .idx rows" - ), + "remote backend unavailable, not recovering overwritten .idx rows on this load" + ); + } else { + match self.repair_idx_head_tombstones(io) { + Ok(0) => {} + Ok(restored) => info!( + volume_id = self.id.0, + restored, "recovered overwritten .idx rows from .dat" + ), + Err(e) => warn!( + volume_id = self.id.0, + error = %e, + "recover overwritten .idx rows" + ), + } } // The permission bits still allow writing on a read-only mount; @@ -2220,21 +2326,15 @@ impl Volume { // ---- SuperBlock I/O ---- fn read_super_block(&self, io: &VolumeIo) -> Result<(), VolumeError> { - let mut header = [0u8; SUPER_BLOCK_SIZE]; - self.read_exact_at_backend(io, &mut header, 0)?; - - let extra_size = u16::from_be_bytes([header[6], header[7]]); - let total_size = SUPER_BLOCK_SIZE + extra_size as usize; - - let mut full_buf = vec![0u8; total_size]; - full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header); - if extra_size > 0 { - self.read_exact_at_backend( - io, - &mut full_buf[SUPER_BLOCK_SIZE..], - SUPER_BLOCK_SIZE as u64, - )?; - } + let full_buf = match io.remote_dat_file.as_ref() { + Some(remote) if self.has_remote_file() => read_remote_super_block( + &self.remote_storage_name_key().0, + remote, + REMOTE_SUPER_BLOCK_TIMEOUT, + REMOTE_SUPER_BLOCK_STALL_PERIOD, + )?, + _ => read_super_block_bytes(|buf, offset| self.read_exact_at_backend(io, buf, offset))?, + }; *self.super_block.write().unwrap() = SuperBlock::from_bytes(&full_buf)?; @@ -11516,6 +11616,254 @@ mod tests { assert!(result.is_err(), "unopenable .dat must fail load"); } + fn silent_tier_backend() -> ( + std::net::TcpListener, + Arc, + ) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let backend = crate::remote_storage::s3_tier::S3TierBackend::new( + &crate::remote_storage::s3_tier::S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket-a".to_string(), + endpoint: format!("http://{}", listener.local_addr().unwrap()), + storage_class: "STANDARD".to_string(), + force_path_style: true, + }, + ); + (listener, Arc::new(backend)) + } + + #[test] + fn remote_super_block_read_gives_up_on_a_silent_backend_and_skips_it() { + let name = "s3.super_block_stall_test"; + let (_listener, backend) = silent_tier_backend(); + let silent = RemoteDatFile { + backend, + key: "remote-key".to_string(), + file_size: 1024, + modified_time: 0, + }; + + let start = Instant::now(); + let err = read_remote_super_block( + name, + &silent, + Duration::from_millis(200), + Duration::from_secs(3600), + ) + .unwrap_err(); + assert!( + matches!(&err, VolumeError::Io(e) if e.kind() == io::ErrorKind::TimedOut), + "{err:?}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + + let start = Instant::now(); + assert!( + read_remote_super_block( + name, + &silent, + Duration::from_secs(30), + Duration::from_secs(3600) + ) + .is_err() + ); + assert!( + start.elapsed() < Duration::from_secs(1), + "a stalled backend must not be asked again within the stall period" + ); + + let sb = SuperBlock { + version: Version::current(), + ..SuperBlock::default() + }; + let (endpoint, shutdown_tx) = spawn_fake_s3_server(sb.to_bytes()); + let answering = RemoteDatFile { + backend: Arc::new(crate::remote_storage::s3_tier::S3TierBackend::new( + &crate::remote_storage::s3_tier::S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket-a".to_string(), + endpoint, + storage_class: "STANDARD".to_string(), + force_path_style: true, + }, + )), + key: "remote-key".to_string(), + file_size: SUPER_BLOCK_SIZE as u64, + modified_time: 0, + }; + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now()); + let bytes = read_remote_super_block( + name, + &answering, + Duration::from_secs(10), + Duration::from_secs(3600), + ) + .unwrap(); + assert_eq!( + SuperBlock::from_bytes(&bytes).unwrap().version, + Version::current() + ); + assert!( + !REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().contains_key(name), + "a successful read clears the stall" + ); + let _ = shutdown_tx.send(()); + } + + #[test] + fn remote_volume_loads_without_waiting_on_a_stalled_backend() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let dat_len = { + let v = make_test_volume(dir); + v.dat_file_size().unwrap() + }; + std::fs::remove_file(format!("{}/1.dat", dir)).unwrap(); + + let (_listener, backend) = silent_tier_backend(); + let name = "s3.load_stall_test"; + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap()); + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now() + Duration::from_secs(3600)); + + let vif = VifVolumeInfo { + files: vec![VifRemoteFile { + backend_type: "s3".to_string(), + backend_id: "load_stall_test".to_string(), + key: "remote-key".to_string(), + offset: 0, + file_size: dat_len, + modified_time: 123, + extension: ".dat".to_string(), + }], + version: Version::current().0 as u32, + replication: "001".to_string(), + ..VifVolumeInfo::default() + }; + std::fs::write( + format!("{}/1.vif", dir), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + + let start = Instant::now(); + let v = Volume::new( + dir, + dir, + VolumeId(1), + NeedleMapKind::InMemory, + &VolumeSpec::default(), + ) + .unwrap(); + assert!(start.elapsed() < Duration::from_secs(5)); + assert!(v.has_remote_file()); + assert_eq!(v.version(), Version::current()); + assert_eq!( + v.super_block.read().unwrap().replica_placement, + ReplicaPlacement::from_string("001").unwrap() + ); + + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .remove(name); + REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name); + } + + #[test] + fn remote_volume_load_skips_idx_repair_on_a_stalled_backend() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let dat_len = { + let v = make_test_volume(dir); + for i in 1..=3u64 { + let mut n = Needle { + id: NeedleId(i), + cookie: Cookie(0x1234), + data: format!("needle-{i}").into_bytes(), + data_size: format!("needle-{i}").len() as u32, + ..Needle::default() + }; + v.write_needle(&mut n, true, false).unwrap(); + } + v.dat_file_size().unwrap() + }; + std::fs::remove_file(format!("{}/1.dat", dir)).unwrap(); + let idx_path = format!("{}/1.idx", dir); + crate::storage::volume_idx_repair::tests::clobber_idx_head(&idx_path, &[4, 5]); + let idx_before = std::fs::read(&idx_path).unwrap(); + + let (_listener, backend) = silent_tier_backend(); + let name = "s3.idx_repair_stall_test"; + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap()); + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now() + Duration::from_secs(3600)); + + let vif = VifVolumeInfo { + files: vec![VifRemoteFile { + backend_type: "s3".to_string(), + backend_id: "idx_repair_stall_test".to_string(), + key: "remote-key".to_string(), + offset: 0, + file_size: dat_len, + modified_time: 123, + extension: ".dat".to_string(), + }], + version: Version::current().0 as u32, + ..VifVolumeInfo::default() + }; + std::fs::write( + format!("{}/1.vif", dir), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + + let start = Instant::now(); + let v = Volume::new( + dir, + dir, + VolumeId(1), + NeedleMapKind::InMemory, + &VolumeSpec::default(), + ) + .unwrap(); + assert!( + start.elapsed() < Duration::from_secs(5), + "the load waited on the stalled backend" + ); + assert!(v.has_remote_file()); + drop(v); + assert_eq!( + std::fs::read(&idx_path).unwrap(), + idx_before, + "the repair ran against an unreachable backend" + ); + + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .remove(name); + REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name); + } + #[test] fn test_remote_only_volume_load_reads_from_tier_backend() { let tmp = TempDir::new().unwrap(); diff --git a/seaweed-volume/src/storage/volume_idx_repair.rs b/seaweed-volume/src/storage/volume_idx_repair.rs index e79e222633..0fd410db8a 100644 --- a/seaweed-volume/src/storage/volume_idx_repair.rs +++ b/seaweed-volume/src/storage/volume_idx_repair.rs @@ -224,7 +224,7 @@ fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::storage::needle::crc::CRC; use crate::storage::needle_map::NeedleMapKind; @@ -267,7 +267,7 @@ mod tests { /// Reproduce the damage a delete on a tiered read-only volume used to do: it /// writes (key, offset 0, tombstone) rows over the front of .idx instead of /// appending them. - fn clobber_idx_head(idx_path: &str, keys: &[u64]) { + pub(crate) fn clobber_idx_head(idx_path: &str, keys: &[u64]) { let mut file = OpenOptions::new().write(true).open(idx_path).unwrap(); for (i, key) in keys.iter().enumerate() { let mut row = Vec::new(); diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go index 08651dedea..689286ad0c 100644 --- a/weed/storage/backend/s3_backend/s3_backend.go +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -1,6 +1,7 @@ package s3_backend import ( + "context" "fmt" "io" "math" @@ -222,7 +223,9 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1) - getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{ + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObjectWithContext(ctx, &s3.GetObjectInput{ Bucket: &s3backendStorageFile.backendStorage.bucket, Key: &s3backendStorageFile.key, Range: &bytesRange, @@ -232,6 +235,8 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr) } defer getObjectOutput.Body.Close() + stalled := time.AfterFunc(s3ReadTimeout, cancel) + defer stalled.Stop() // glog.V(3).Infof("read %s %s", s3backendStorageFile.key, bytesRange) // glog.V(3).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength) @@ -242,6 +247,7 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n if len(p) == 0 { break } + stalled.Reset(s3ReadTimeout) readCount, err = getObjectOutput.Body.Read(p) n += readCount @@ -253,6 +259,9 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n if err == io.EOF && n == requested { err = nil } + if err != nil && ctx.Err() != nil { + err = fmt.Errorf("bucket %s read %s: no data for %v: %w", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, s3ReadTimeout, err) + } return } diff --git a/weed/storage/backend/s3_backend/s3_backend_test.go b/weed/storage/backend/s3_backend/s3_backend_test.go index 347c379820..f103469ad0 100644 --- a/weed/storage/backend/s3_backend/s3_backend_test.go +++ b/weed/storage/backend/s3_backend/s3_backend_test.go @@ -7,6 +7,8 @@ import ( "testing" "testing/iotest" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -23,7 +25,7 @@ type stubS3Client struct { getObject func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) } -func (s *stubS3Client) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { +func (s *stubS3Client) GetObjectWithContext(_ aws.Context, input *s3.GetObjectInput, _ ...request.Option) (*s3.GetObjectOutput, error) { return s.getObject(input) } diff --git a/weed/storage/backend/s3_backend/s3_sessions.go b/weed/storage/backend/s3_backend/s3_sessions.go index bdadb56118..bf9f3af5f0 100644 --- a/weed/storage/backend/s3_backend/s3_sessions.go +++ b/weed/storage/backend/s3_backend/s3_sessions.go @@ -3,6 +3,7 @@ package s3_backend import ( "crypto/tls" "fmt" + "net" "net/http" "sync" "time" @@ -21,8 +22,14 @@ var ( sessionsLock sync.Mutex ) -// httpOptions tunes the HTTP transport of the tier backend's S3 client. The -// zero value keeps the SDK's default client. +// A tier endpoint that drops packets or accepts connections without answering +// must not hold a read, or a volume server's startup, for minutes per request. +var ( + s3ConnectTimeout = 10 * time.Second + s3ReadTimeout = time.Minute +) + +// httpOptions tunes the HTTP transport of the tier backend's S3 client. type httpOptions struct { // disableHTTP2 keeps the client on HTTP/1.1. Over HTTP/2 every upload and // range read to the endpoint shares one connection and its flow-control @@ -36,12 +43,10 @@ type httpOptions struct { idleConnTimeout time.Duration } -// newTransport returns the transport for the options, or nil for the default. func (o httpOptions) newTransport() *http.Transport { - if o == (httpOptions{}) { - return nil - } t := http.DefaultTransport.(*http.Transport).Clone() + t.DialContext = (&net.Dialer{Timeout: s3ConnectTimeout, KeepAlive: 30 * time.Second}).DialContext + t.ResponseHeaderTimeout = s3ReadTimeout if o.disableHTTP2 { t.ForceAttemptHTTP2 = false t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} @@ -73,9 +78,7 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string, Endpoint: aws.String(endpoint), S3ForcePathStyle: aws.Bool(forcePathStyle), S3DisableContentMD5Validation: aws.Bool(true), - } - if t := httpOpts.newTransport(); t != nil { - config.HTTPClient = &http.Client{Transport: t} + HTTPClient: &http.Client{Transport: httpOpts.newTransport()}, } if awsAccessKeyId != "" && awsSecretAccessKey != "" { config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") diff --git a/weed/storage/backend/s3_backend/s3_sessions_test.go b/weed/storage/backend/s3_backend/s3_sessions_test.go index eb9f621c4d..eb5699259c 100644 --- a/weed/storage/backend/s3_backend/s3_sessions_test.go +++ b/weed/storage/backend/s3_backend/s3_sessions_test.go @@ -1,13 +1,18 @@ package s3_backend import ( + "bufio" "math" + "net" "net/http" "strconv" + "strings" + "sync" "testing" "time" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" ) func backendTransport(t *testing.T, s *S3BackendStorage) *http.Transport { @@ -54,17 +59,103 @@ func TestS3BackendHTTPOptionsReachTheClient(t *testing.T) { } } -func TestS3BackendDefaultHTTPClientUnchanged(t *testing.T) { +func TestS3BackendDefaultTransportHasTimeouts(t *testing.T) { s, err := newS3BackendStorage(testProperties{"endpoint": "http://127.0.0.1:2"}, "", "http-defaults") if err != nil { t.Fatal(err) } - if transport := backendTransport(t, s); transport != nil && transport != http.DefaultTransport { - t.Fatalf("no options set, yet the client got a custom transport %+v", transport) + transport := backendTransport(t, s) + if transport == nil || transport == http.DefaultTransport { + t.Fatalf("the client must not use the shared default transport, got %+v", transport) } - if (httpOptions{}).newTransport() != nil { - t.Fatal("zero options must keep the default client") + if transport.ResponseHeaderTimeout != s3ReadTimeout { + t.Errorf("ResponseHeaderTimeout = %v, want %v", transport.ResponseHeaderTimeout, s3ReadTimeout) } + if transport.DialContext == nil { + t.Error("no dialer with a connect timeout") + } + if !transport.ForceAttemptHTTP2 { + t.Error("default options must keep HTTP/2") + } +} + +// stallingEndpoint accepts connections and holds them open. With a non-empty +// head, it answers each request with head and then sends nothing more. +func stallingEndpoint(t *testing.T, head string) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var conns []net.Conn + var mu sync.Mutex + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + if head != "" { + go func() { + if _, err := http.ReadRequest(bufio.NewReader(c)); err == nil { + c.Write([]byte(head)) + } + }() + } + } + }() + t.Cleanup(func() { + ln.Close() + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + }) + return "http://" + ln.Addr().String() +} + +func readFromStallingEndpoint(t *testing.T, head, wantErr string) { + old := s3ReadTimeout + s3ReadTimeout = 100 * time.Millisecond + defer func() { s3ReadTimeout = old }() + + s, err := newS3BackendStorage(testProperties{ + "endpoint": stallingEndpoint(t, head), + "region": "us-east-1", + "bucket": "b", + "aws_access_key_id": "k", + "aws_secret_access_key": "s", + }, "", "stalling") + if err != nil { + t.Fatal(err) + } + f := s.NewStorageFile("key", &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{{FileSize: 1024}}}) + + done := make(chan error, 1) + go func() { + _, err := f.ReadAt(make([]byte, 8), 0) + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("want an error containing %q from a stalled endpoint, got %v", wantErr, err) + } + case <-time.After(30 * time.Second): + t.Fatal("a read from a stalled endpoint did not give up") + } +} + +func TestS3BackendReadFailsAgainstSilentEndpoint(t *testing.T) { + readFromStallingEndpoint(t, "", "timeout") +} + +func TestS3BackendReadFailsWhenBodyStalls(t *testing.T) { + readFromStallingEndpoint(t, "HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n", "no data for") } func TestS3BackendSessionsDifferByHTTPOptions(t *testing.T) { diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index ae2fd07c1b..6e464c0845 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -232,6 +232,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind return datFileLoadError(v.FileName(".dat"), err) } + var remoteSuperBlockErr error if alreadyHasSuperBlock { err = v.readSuperBlock() if err == nil { @@ -244,6 +245,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind if v.HasRemoteFile() { // maybe temporary network problem logRemoteSuperBlockErr(v.Id, err) + remoteSuperBlockErr = err err = nil } } else { @@ -292,7 +294,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind // Recover rows that deletes on a tiered read-only volume overwrote at // the front of .idx. Best effort: a volume that cannot be repaired is // still servable for everything the surviving rows index. - if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil { + if remoteSuperBlockErr != nil { + glog.V(1).Infof("volume %d: remote backend unavailable, not recovering overwritten %s rows on this load", v.Id, v.FileName(".idx")) + } else if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil { glog.Warningf("volume %d: recover overwritten %s rows: %v", v.Id, v.FileName(".idx"), repairErr) } else if restored > 0 { glog.V(0).Infof("volume %d: recovered %d overwritten %s rows from %s", v.Id, restored, v.FileName(".idx"), v.FileName(".dat")) diff --git a/weed/storage/volume_remote_superblock_stall_test.go b/weed/storage/volume_remote_superblock_stall_test.go new file mode 100644 index 0000000000..4aa43a7c35 --- /dev/null +++ b/weed/storage/volume_remote_superblock_stall_test.go @@ -0,0 +1,185 @@ +package storage + +import ( + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// stallingBackend serves localDirBackend's objects, but every read waits +// until release is closed: an endpoint that accepts requests and never answers. +type stallingBackend struct { + *localDirBackend + release chan struct{} + reads atomic.Int32 +} + +func (b *stallingBackend) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile { + return &stallingBackendFile{BackendStorageFile: b.localDirBackend.NewStorageFile(key, tierInfo), backend: b} +} + +type stallingBackendFile struct { + backend.BackendStorageFile + backend *stallingBackend +} + +func (f *stallingBackendFile) ReadAt(p []byte, off int64) (int, error) { + f.backend.reads.Add(1) + <-f.backend.release + return f.BackendStorageFile.ReadAt(p, off) +} + +func shortenRemoteSuperBlockWait(t *testing.T, timeout, stall time.Duration) { + t.Helper() + oldTimeout, oldStall := remoteSuperBlockTimeout, remoteSuperBlockStallPeriod + remoteSuperBlockTimeout, remoteSuperBlockStallPeriod = timeout, stall + remoteSuperBlockStalls.Delete(testBackendName) + t.Cleanup(func() { + remoteSuperBlockTimeout, remoteSuperBlockStallPeriod = oldTimeout, oldStall + remoteSuperBlockStalls.Delete(testBackendName) + }) +} + +func TestReadRemoteSuperBlockStallSkipsBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 50*time.Millisecond, time.Hour) + b := &stallingBackend{localDirBackend: newLocalDirBackend(t), release: make(chan struct{})} + key, _, err := b.CopyFile(writeSuperBlockFile(t), nil, 0) + require.NoError(t, err) + f := b.NewStorageFile(key, &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{{FileSize: super_block.SuperBlockSize}}}) + + start := time.Now() + _, err = readRemoteSuperBlock(testBackendName, f) + require.Error(t, err) + require.Less(t, time.Since(start), 5*time.Second) + require.EqualValues(t, 1, b.reads.Load()) + + _, err = readRemoteSuperBlock(testBackendName, f) + require.Error(t, err) + require.EqualValues(t, 1, b.reads.Load(), "a stalled backend must not be asked again within the stall period") + + close(b.release) + remoteSuperBlockStalls.Store(testBackendName, time.Now().Add(-time.Second)) + sb, err := readRemoteSuperBlock(testBackendName, f) + require.NoError(t, err) + require.Equal(t, needle.GetCurrentVersion(), sb.Version) + _, stalled := remoteSuperBlockStalls.Load(testBackendName) + require.False(t, stalled, "a successful read clears the stall") +} + +func writeSuperBlockFile(t *testing.T) *os.File { + t.Helper() + f, err := os.Create(filepath.Join(t.TempDir(), "sb.dat")) + require.NoError(t, err) + t.Cleanup(func() { f.Close() }) + sb := super_block.SuperBlock{Version: needle.GetCurrentVersion(), ReplicaPlacement: &super_block.ReplicaPlacement{}, Ttl: &needle.TTL{}} + _, err = f.Write(sb.Bytes()) + require.NoError(t, err) + return f +} + +// A backend that never answers must not hold up the disk scan, local volumes +// included, and the tiered volumes must serve reads once it answers again. +func TestRemoteTier_DiskScanDoesNotWaitOnStalledBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 200*time.Millisecond, time.Hour) + local := newLocalDirBackend(t) + registerTestBackend(t, local) + + dir := t.TempDir() + remoteVids := []needle.VolumeId{61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72} + for _, vid := range remoteVids { + tierUpVolume(t, dir, vid, local) + } + lv, err := NewVolume(dir, dir, "", 60, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + require.NoError(t, err) + lv.Close() + + stalling := &stallingBackend{localDirBackend: local, release: make(chan struct{})} + backend.BackendStorages[testBackendName] = stalling + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(stalling.release) }) } + t.Cleanup(release) + + loc := &DiskLocation{ + Directory: dir, + DirectoryUuid: "test-uuid", + IdxDirectory: dir, + DiskType: types.HddType, + MaxVolumeCount: 100, + OriginalMaxVolumeCount: 100, + MinFreeSpace: util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}, + } + loc.volumes = make(map[needle.VolumeId]*Volume) + loc.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) + + start := time.Now() + loc.concurrentLoadingVolumes(NeedleMapInMemory, 2, 0, 0) + require.Less(t, time.Since(start), 5*time.Second, "the disk scan waited on the stalled backend") + defer func() { + for _, v := range loc.volumes { + v.Close() + } + }() + + _, ok := loc.FindVolume(60) + require.True(t, ok, "local volume not loaded") + for _, vid := range remoteVids { + v, ok := loc.FindVolume(vid) + require.True(t, ok, "tiered volume %d not loaded", vid) + require.True(t, v.HasRemoteFile()) + } + require.Less(t, int(stalling.reads.Load()), len(remoteVids), "every tiered volume waited on the stalled backend") + + release() + v, _ := loc.FindVolume(remoteVids[0]) + _, err = v.readNeedle(newEmptyNeedle(1), nil, nil) + require.NoError(t, err, "a tiered volume must serve reads once the backend answers") +} + +// The .idx head repair reads the remote .dat, so it must wait for a load that +// can reach the backend instead of stalling this one. +func TestRemoteTier_IdxHeadRepairWaitsForReachableBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 200*time.Millisecond, time.Hour) + local := newLocalDirBackend(t) + registerTestBackend(t, local) + + dir := t.TempDir() + const vid = needle.VolumeId(80) + tierUpVolume(t, dir, vid, local) + idxPath := filepath.Join(dir, "80.idx") + clobberIdxHead(t, idxPath, []uint64{4, 5}) + sizeBefore := fileSize(t, idxPath) + + stalling := &stallingBackend{localDirBackend: local, release: make(chan struct{})} + backend.BackendStorages[testBackendName] = stalling + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(stalling.release) }) } + time.AfterFunc(10*time.Second, release) + t.Cleanup(release) + + start := time.Now() + v := reloadVolume(t, dir, vid) + require.Less(t, time.Since(start), 5*time.Second, "the load waited on the stalled backend") + v.Close() + require.Equal(t, sizeBefore, fileSize(t, idxPath), "the repair ran against an unreachable backend") + + backend.BackendStorages[testBackendName] = local + remoteSuperBlockStalls.Delete(testBackendName) + v = reloadVolume(t, dir, vid) + defer v.Close() + require.Greater(t, fileSize(t, idxPath), sizeBefore, "the repair did not run once the backend answered") + _, err := v.readNeedle(newEmptyNeedle(1), nil, nil) + require.NoError(t, err) +} diff --git a/weed/storage/volume_super_block.go b/weed/storage/volume_super_block.go index 7035d9867c..b6faf0ea07 100644 --- a/weed/storage/volume_super_block.go +++ b/weed/storage/volume_super_block.go @@ -3,6 +3,8 @@ package storage import ( "fmt" "os" + "sync" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/backend" @@ -41,7 +43,11 @@ func (v *Volume) maybeWriteSuperBlock(ver needle.Version) error { } func (v *Volume) readSuperBlock() (err error) { - v.SuperBlock, err = super_block.ReadSuperBlock(v.DataBackend) + if v.HasRemoteFile() && len(v.volumeInfo.GetFiles()) > 0 { + v.SuperBlock, err = readRemoteSuperBlock(v.volumeInfo.GetFiles()[0].BackendName(), v.DataBackend) + } else { + v.SuperBlock, err = super_block.ReadSuperBlock(v.DataBackend) + } if v.volumeInfo != nil && v.volumeInfo.Replication != "" { if replication, err := super_block.NewReplicaPlacementFromString(v.volumeInfo.Replication); err != nil { return fmt.Errorf("Error parse volume %d replication %s : %v", v.Id, v.volumeInfo.Replication, err) @@ -51,3 +57,38 @@ func (v *Volume) readSuperBlock() (err error) { } return err } + +// A tiered volume loads without its superblock when the backend cannot serve +// it; once a read stalls, the backend's other volumes skip it for a while. +var ( + remoteSuperBlockTimeout = 10 * time.Second + remoteSuperBlockStallPeriod = time.Minute + remoteSuperBlockStalls sync.Map +) + +func readRemoteSuperBlock(backendName string, datBackend backend.BackendStorageFile) (super_block.SuperBlock, error) { + if until, ok := remoteSuperBlockStalls.Load(backendName); ok && time.Now().Before(until.(time.Time)) { + return super_block.SuperBlock{}, fmt.Errorf("backend %s stalled on a super block read, skipped", backendName) + } + type result struct { + superBlock super_block.SuperBlock + err error + } + done := make(chan result, 1) + go func() { + sb, err := super_block.ReadSuperBlock(datBackend) + done <- result{sb, err} + }() + timer := time.NewTimer(remoteSuperBlockTimeout) + defer timer.Stop() + select { + case r := <-done: + if r.err == nil { + remoteSuperBlockStalls.Delete(backendName) + } + return r.superBlock, r.err + case <-timer.C: + remoteSuperBlockStalls.Store(backendName, time.Now().Add(remoteSuperBlockStallPeriod)) + return super_block.SuperBlock{}, fmt.Errorf("backend %s did not return the super block within %v", backendName, remoteSuperBlockTimeout) + } +} From 533f5060b932ea6d0379f292b8931c320adfbd5c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:36:31 -0800 Subject: [PATCH 08/31] filer: scope a prefix JWT to the directory a zip extract writes under (#12305) * filer: scope a prefix JWT to the directory a zip extract writes under Without a trailing slash, an extract request writes members beside the named path, in its parent directory, while the JWT check only saw the request path. A token scoped to /tenant1 could therefore extract into / by posting to /tenant1?extract=zip. Add the extract root to the paths a prefix-restricted token must cover. Co-Authored-By: Claude Opus 5.5 * filer: do not keep an extracted archive under the name ".." A multipart file name of ".." passes path.Base unchanged, so extract.keep stored the archive as an entry literally named ".." in the target directory, a name that local sinks resolve to the parent. Co-Authored-By: Claude Opus 5.5 * leveldb3: keep a non-local bucket name out of the per-bucket folder findDB splits the bucket out of a /buckets/ path and uses it as a folder name under the store directory. The name cannot hold a slash and dot names already go to the default DB, so this only rejects an empty or non-local name, but it also lets static analysis see the folder cannot escape the store directory. * filer: clean the zip extract root before scoping and writing A percent-encoded dot segment survives the path cleaning in front of the mux, so a request path ending in "/" reached extractZipRoot with ".." in it. The JWT scope check cleans the path it compares, but the members were written under the raw root, so the two could disagree. * leveldb3: apply the bucket folder check to bucket create and delete events findDB already keeps dot and non-local bucket names out of the per-bucket folders, but OnBucketCreation and OnBucketDeletion used the name as is. A directory event for /buckets/.. opened a DB in the parent of the store directory, and its delete removed that parent. Co-Authored-By: Claude Opus 5.5 * leveldb3: keep a bucket delete event off the default DB A bucket named _main maps to the default DB in findDB, so deleting /buckets/_main closed the default DB and removed its folder, taking every non-bucket entry with it. Co-Authored-By: Claude Opus 5.5 * leveldb3: keep bucket names with a path separator out of the store dir A bucket event named x/../_main passed the local-name check, and its delete removed the default DB folder once store.dir/x existed. Co-Authored-By: Claude Opus 5.5 * leveldb3: only reject the platform separator in a bucket name A backslash is an ordinary file name character on Unix, and a bucket folder named that way already exists for anyone who created one. Rejecting it on every platform would route those entries to the default DB after an upgrade. The slash can never appear in a bucket name here, and on Windows the backslash is the separator, so rejecting os.PathSeparator alongside the slash keeps the folder flat without changing Unix routing. * leveldb3: check for an escaped bucket db before its delete event --------- Co-authored-by: Claude Opus 5.5 --- weed/filer/leveldb3/leveldb3_store.go | 13 +++- weed/filer/leveldb3/leveldb3_store_bucket.go | 10 ++- weed/filer/leveldb3/leveldb3_store_test.go | 43 +++++++++++++ weed/server/filer_jwt_test.go | 61 +++++++++++++++++++ weed/server/filer_server_handlers.go | 8 ++- weed/server/filer_server_handlers_extract.go | 19 ++++-- .../filer_server_handlers_extract_test.go | 44 +++++++++++++ 7 files changed, 184 insertions(+), 14 deletions(-) diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 3111e2a5ad..5484383713 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -8,6 +8,7 @@ import ( "io" "math" "os" + "path/filepath" "strings" "sync" @@ -167,9 +168,7 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo shortPath = weed_util.FullPath(bucketAndObjectKey[t:]) } - // Dot-prefixed entries directly under /buckets (e.g. .system) are internal - // folders, not S3 buckets; keep them in the default DB by full path. - if strings.HasPrefix(bucket, ".") { + if !isBucketFolder(bucket) { store.dbsLock.RUnlock() return defaultDB, DEFAULT, fullpath, nil } @@ -207,6 +206,14 @@ func (store *LevelDB3Store) emptyBucketDB() (*leveldb.DB, error) { return store.emptyDB, nil } +// Dot-prefixed entries directly under /buckets (e.g. .system) are internal +// folders, not S3 buckets; they stay in the default DB by full path. A bucket +// also names a single folder under store.dir, so it cannot hold a separator; +// "\" is an ordinary name character on Unix and only rejected where it separates. +func isBucketFolder(bucket string) bool { + return !strings.HasPrefix(bucket, ".") && filepath.IsLocal(bucket) && !strings.ContainsAny(bucket, "/"+string(os.PathSeparator)) +} + func (store *LevelDB3Store) createDB(bucket string) (*leveldb.DB, error) { store.dbsLock.Lock() diff --git a/weed/filer/leveldb3/leveldb3_store_bucket.go b/weed/filer/leveldb3/leveldb3_store_bucket.go index feb2a4b049..2b56cad6ca 100644 --- a/weed/filer/leveldb3/leveldb3_store_bucket.go +++ b/weed/filer/leveldb3/leveldb3_store_bucket.go @@ -9,14 +9,18 @@ import ( var _ filer.BucketAware = (*LevelDB3Store)(nil) func (store *LevelDB3Store) OnBucketCreation(bucket string) { + if !isBucketFolder(bucket) { + return + } store.createDB(bucket) } func (store *LevelDB3Store) OnBucketDeletion(bucket string) { - store.closeDB(bucket) - if bucket != "" { // just to make sure - os.RemoveAll(store.dir + "/" + bucket) + if !isBucketFolder(bucket) || bucket == DEFAULT { + return } + store.closeDB(bucket) + os.RemoveAll(store.dir + "/" + bucket) } func (store *LevelDB3Store) CanDropWholeBucket() bool { diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index a9f3be10da..1f57acd4c1 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -3,6 +3,8 @@ package leveldb import ( "context" "math" + "os" + "path/filepath" "strings" "testing" @@ -207,3 +209,44 @@ func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) { t.Fatalf("want [item-b.versions], got %v", names) } } + +func TestBucketEventsStayInsideStoreDir(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "store") + store := &LevelDB3Store{} + if err := store.initialize(dir); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + + sentinel := filepath.Join(parent, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + // Checked after each create, before the matching delete could remove the + // evidence of a db opened in the wrong place. + assertNothingEscaped := func(bucket string) { + t.Helper() + for _, p := range []string{filepath.Join(parent, "outside"), filepath.Join(parent, "CURRENT"), filepath.Join(dir, "x", "y")} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("bucket %q event created %s: %v", bucket, p, err) + } + } + } + store.OnBucketCreation("x") + for _, bucket := range []string{"..", ".", "../outside", "", DEFAULT, "x/../" + DEFAULT, "x/y"} { + store.OnBucketCreation(bucket) + assertNothingEscaped(bucket) + store.OnBucketDeletion(bucket) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("bucket events removed %s: %v", sentinel, err) + } + if _, err := os.Stat(filepath.Join(dir, DEFAULT)); err != nil { + t.Fatalf("default db folder missing: %v", err) + } + ctx := context.Background() + if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/a/b"}); err != nil { + t.Fatalf("default db unusable after bucket events: %v", err) + } +} diff --git a/weed/server/filer_jwt_test.go b/weed/server/filer_jwt_test.go index 74f5bb9bcc..efd4b99c60 100644 --- a/weed/server/filer_jwt_test.go +++ b/weed/server/filer_jwt_test.go @@ -38,6 +38,7 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { token string method string path string + header map[string]string isWrite bool expectAuthorized bool }{ @@ -225,6 +226,63 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { isWrite: true, expectAuthorized: false, }, + { + name: "write at the prefix itself allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract into the prefix directory allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1/?extract=zip", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract beside an archive inside the prefix allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "PUT", + path: "/tenant1/pack.zip?extract=zip", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract at the prefix itself writes to its parent, denied", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1?extract=zip", + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract header at the prefix itself denied", + token: genToken([]string{"/tenant1/"}, []string{"PUT"}), + method: "PUT", + path: "/tenant1", + header: map[string]string{"X-Seaweedfs-Extract": "zip"}, + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract at a nested prefix into the sibling parent denied", + token: genToken([]string{"/buckets/bucket-a"}, nil), + method: "POST", + path: "/buckets/bucket-a?extract=zip", + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract with a root prefix allowed", + token: genToken([]string{"/"}, nil), + method: "POST", + path: "/tenant1?extract=zip", + isWrite: true, + expectAuthorized: true, + }, } for _, tt := range tests { @@ -233,6 +291,9 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { if tt.token != "" { req.Header.Set("Authorization", "Bearer "+tt.token) } + for k, v := range tt.header { + req.Header.Set(k, v) + } if authorized := fs.maybeCheckJwtAuthorization(req, tt.isWrite); authorized != tt.expectAuthorized { t.Errorf("expected authorized=%v, got %v", tt.expectAuthorized, authorized) } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 6fd9bfbf3e..4bdd5d87f9 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -308,10 +308,14 @@ func authorizeFilerJwtPaths(r *http.Request, claims *security.SeaweedFilerClaims } // jwtScopedRequestPaths returns every filer path a request touches that must be -// covered by the JWT AllowedPrefixes: the write target (r.URL.Path) plus any -// copy/move source named by the cp.from / mv.from query parameters. +// covered by the JWT AllowedPrefixes: the write target (r.URL.Path), the +// directory a zip extract writes under, and any copy/move source named by the +// cp.from / mv.from query parameters. func jwtScopedRequestPaths(r *http.Request) []string { paths := []string{r.URL.Path} + if (r.Method == http.MethodPost || r.Method == http.MethodPut) && isExtractZip(r) { + paths = append(paths, extractZipRoot(r)+"/") + } if query := r.URL.Query(); query.Has("cp.from") || query.Has("mv.from") { if from := query.Get("cp.from"); from != "" { paths = append(paths, from) diff --git a/weed/server/filer_server_handlers_extract.go b/weed/server/filer_server_handlers_extract.go index c51d7b9c6f..97057ee3fc 100644 --- a/weed/server/filer_server_handlers_extract.go +++ b/weed/server/filer_server_handlers_extract.go @@ -35,6 +35,17 @@ func isExtractZip(r *http.Request) bool { r.Header.Get(s3_constants.SeaweedFSExtract) == "zip" } +// extractZipRoot is the directory members land under: the request path when it +// ends in "/", otherwise its parent, so the path can name the archive. It is +// cleaned so the JWT scope check and the writes agree on dot segments. +func extractZipRoot(r *http.Request) string { + root := r.URL.Path + if !strings.HasSuffix(root, "/") { + root = path.Dir(root) + } + return strings.TrimSuffix(path.Clean(root), "/") +} + // extractZipArchive writes a posted archive's members as individual entries // under the target directory, like AWS Snowball's auto-extract. The archive // itself is only stored when ?extract.keep=true. @@ -81,11 +92,7 @@ func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWri return } - root := r.URL.Path - if !strings.HasSuffix(root, "/") { - root = path.Dir(root) - } - root = strings.TrimSuffix(root, "/") + root := extractZipRoot(r) query := r.URL.Query() chunkSize := fs.requestChunkSize(query) @@ -153,7 +160,7 @@ func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWri } } - if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != "/" { + if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != ".." && archiveName != "/" { mode := os.FileMode(0660) if modeStr := query.Get("mode"); modeStr != "" { if parsed, parseErr := strconv.ParseUint(modeStr, 8, 32); parseErr == nil { diff --git a/weed/server/filer_server_handlers_extract_test.go b/weed/server/filer_server_handlers_extract_test.go index 9461e79e5a..3040a9ad4a 100644 --- a/weed/server/filer_server_handlers_extract_test.go +++ b/weed/server/filer_server_handlers_extract_test.go @@ -168,6 +168,28 @@ func TestExtractZipArchiveKeep(t *testing.T) { } } +func TestExtractZipArchiveKeepSkipsParentName(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"x.txt": "data"}, nil, nil) + + var body bytes.Buffer + mw := multipartWriter(t, &body, "..", zipBytes) + r := httptest.NewRequest(http.MethodPost, "/a/b/?extract=zip&extract.keep=true", &body) + r.Header.Set("Content-Type", mw) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(body.Len())) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/a/b/..")); err == nil { + t.Fatal("archive kept under a parent-directory name") + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/a/b/x.txt")); err != nil { + t.Fatalf("member missing: %v", err) + } +} + func TestExtractZipArchiveNotAZip(t *testing.T) { fs, _ := newExtractTestServer(t) r := httptest.NewRequest(http.MethodPut, "/dst/notzip.bin?extract=zip", bytes.NewReader([]byte("not a zip"))) @@ -231,6 +253,28 @@ func TestExtractZipArchiveRawPost(t *testing.T) { } } +func TestExtractZipArchiveCleansDotSegments(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"raw.txt": "raw"}, nil, nil) + + // A percent-encoded ".." survives path cleaning in front of the mux, so the + // handler sees it decoded and must land members under the cleaned root. + r := httptest.NewRequest(http.MethodPost, "/other/%2e%2e/dst/?extract=zip", bytes.NewReader(zipBytes)) + r.Header.Set("Content-Type", "application/zip") + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/raw.txt")); err != nil { + t.Fatalf("member missing under the cleaned root: %v", err) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/other/../dst/raw.txt")); err == nil { + t.Fatal("member written under the raw root") + } +} + func TestExtractZipArchiveMultipartPost(t *testing.T) { fs, f := newExtractTestServer(t) zipBytes := buildTestZip(t, map[string]string{"img/1.png": "png-bytes"}, nil, nil) From 06b9d014237b5709cf2d218d879599117be56281 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:44:49 -0800 Subject: [PATCH 09/31] s3vectors: match a filter value against the elements of an array (#12321) * vector: compare json_array_contains elements as typed JSON values Lance's json_array_contains compares each element's text with the value, or with the value wrapped in quotes, so 1 matches "1", true matches "true", and 1.0 misses a stored 1. The filter swap now replaces it like json_extract and json_exists: the value is parsed as JSON once per batch and compared with jsonb's typed equality, and only literal arguments are swapped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3vectors: match a filter value against the elements of an array A filter never looked inside an array: {"tags": "space"} and {"tags": {"$in": ["space"]}} both missed "tags": ["space", "classic"]. AWS documents that $eq, and so a bare value, matches an array metadata value when any element equals the input. $eq, a bare value and each $in member now also match through json_array_contains; $ne and $nin remain their negations. Range operators, $exists and nested objects are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- seaweed-vector/crates/core/src/filter.rs | 127 +++++++++++++++++++---- test/s3vectors/s3vectors_test.py | 19 ++++ weed/s3api/s3vectors/handlers.go | 52 ++++++++-- weed/s3api/s3vectors/handlers_test.go | 31 ++++-- 4 files changed, 190 insertions(+), 39 deletions(-) diff --git a/seaweed-vector/crates/core/src/filter.rs b/seaweed-vector/crates/core/src/filter.rs index da1fabf458..9613e9cafa 100644 --- a/seaweed-vector/crates/core/src/filter.rs +++ b/seaweed-vector/crates/core/src/filter.rs @@ -1,14 +1,16 @@ //! Lance's json_extract and json_exists parse their JSONPath again for every //! row. A filter's path is a literal, so swap in copies that parse it once per -//! batch and return the same values. +//! batch and return the same values. json_array_contains is swapped too, and +//! compares the elements as typed JSON values: Lance's compares their text, so +//! 1 would match "1". use std::sync::Arc; use arrow_array::builder::{BooleanBuilder, StringBuilder}; use arrow_array::{Array, ArrayRef, LargeBinaryArray, new_null_array}; use arrow_schema::DataType; -use jsonb::RawJsonb; use jsonb::jsonpath::{Selector, parse_json_path}; +use jsonb::{OwnedJsonb, RawJsonb}; use lance::dataset::scanner::Scanner; use lance::deps::datafusion::common::tree_node::{Transformed, TreeNode}; use lance::deps::datafusion::common::{DataFusionError, Result, ScalarValue}; @@ -34,12 +36,16 @@ fn swap(expr: Expr) -> Result { let op = match f.func.name() { "json_extract" => Op::Extract, "json_exists" => Op::Exists, + "json_array_contains" => Op::Contains, _ => return Ok(Transformed::no(Expr::ScalarFunction(f))), }; - if !matches!( - f.args.get(1), - Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _)) - ) { + let literal = |i: usize| { + matches!( + f.args.get(i), + Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _)) + ) + }; + if !literal(1) || (op == Op::Contains && !literal(2)) { return Ok(Transformed::no(Expr::ScalarFunction(f))); } let udf = Arc::new(ScalarUDF::new_from_impl(JsonPath::new(op))); @@ -54,6 +60,7 @@ fn swap(expr: Expr) -> Result { enum Op { Extract, Exists, + Contains, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -64,12 +71,13 @@ struct JsonPath { impl JsonPath { fn new(op: Op) -> Self { + let mut args = vec![DataType::LargeBinary, DataType::Utf8]; + if op == Op::Contains { + args.push(DataType::Utf8); + } Self { op, - signature: Signature::exact( - vec![DataType::LargeBinary, DataType::Utf8], - Volatility::Immutable, - ), + signature: Signature::exact(args, Volatility::Immutable), } } } @@ -83,6 +91,7 @@ impl ScalarUDFImpl for JsonPath { match self.op { Op::Extract => "json_extract", Op::Exists => "json_exists", + Op::Contains => "json_array_contains", } } @@ -93,17 +102,24 @@ impl ScalarUDFImpl for JsonPath { fn return_type(&self, _: &[DataType]) -> Result { Ok(match self.op { Op::Extract => DataType::Utf8, - Op::Exists => DataType::Boolean, + Op::Exists | Op::Contains => DataType::Boolean, }) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] = args.args.as_slice() - else { - return Err(exec_err(format!( - "{} expects a literal JSONPath", - self.name() - ))); + let (data, path, value) = match args.args.as_slice() { + [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] => (data, path, None), + [ + data, + ColumnarValue::Scalar(ScalarValue::Utf8(Some(path))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(value))), + ] => (data, path, Some(value)), + _ => { + return Err(exec_err(format!( + "{} expects literal arguments", + self.name() + ))); + } }; let data = data.to_array(args.number_rows)?; let data = data @@ -160,6 +176,37 @@ impl ScalarUDFImpl for JsonPath { } Arc::new(builder.finish()) } + Op::Contains => { + let value = value.map(String::as_str).unwrap_or_default(); + let value: OwnedJsonb = value + .parse() + .map_err(|e| exec_err(format!("Invalid JSON value '{value}': {e}")))?; + let mut builder = BooleanBuilder::with_capacity(data.len()); + for i in 0..data.len() { + if data.is_null(i) { + builder.append_null(); + continue; + } + let mut selector = Selector::new(RawJsonb::new(data.value(i))); + let found = match selector.select_value(&json_path) { + Ok(Some(array)) => array + .as_raw() + .array_values() + .map_err(|e| { + exec_err(format!("Failed to read array at '{path}': {e}")) + })? + .is_some_and(|items| items.contains(&value)), + Ok(None) => false, + Err(e) => { + return Err(exec_err(format!( + "Failed to check array contains at path '{path}': {e}" + ))); + } + }; + builder.append_value(found); + } + Arc::new(builder.finish()) + } }; Ok(ColumnarValue::Array(out)) } @@ -298,6 +345,48 @@ mod tests { assert!(keys(&dataset, bad, true).await.is_err()); } + #[tokio::test] + async fn array_contains_compares_typed_values() { + let dir = TempDir::new().unwrap(); + let metas = [ + Some(r#"{"k": ["space", "classic"]}"#), + Some(r#"{"k": [1, 2.5]}"#), + Some(r#"{"k": ["1", "2.5"]}"#), + Some(r#"{"k": [true]}"#), + Some(r#"{"k": ["true"]}"#), + Some(r#"{"k": "space"}"#), + Some(r#"{"k": [["space"]]}"#), + Some(r#"{"k": {"a": "space"}}"#), + Some(r#"{"k": ["", "spa\"ce"]}"#), + Some(r#"{"k": []}"#), + Some(r#"{"x": 1}"#), + None, + ]; + let dataset = dataset(&dir, &metas).await; + for (value, want) in [ + (r#""space""#, vec!["k0"]), + ("1", vec!["k1"]), + ("1.0", vec!["k1"]), + ("2.5", vec!["k1"]), + (r#""1""#, vec!["k2"]), + ("true", vec!["k3"]), + (r#""true""#, vec!["k4"]), + (r#""""#, vec!["k8"]), + (r#""spa\"ce""#, vec!["k8"]), + (r#""classic""#, vec!["k0"]), + ("3", vec![]), + ] { + let filter = format!(r#"json_array_contains(`metadata`, '$["k"]', '{value}')"#); + assert_eq!( + keys(&dataset, &filter, true).await.unwrap(), + want, + "{filter}" + ); + } + let bad = r#"json_array_contains(`metadata`, '$["k"]', 'not json')"#; + assert!(keys(&dataset, bad, true).await.is_err()); + } + #[tokio::test] async fn apply_swaps_literal_paths_only() { let dir = TempDir::new().unwrap(); @@ -305,7 +394,7 @@ mod tests { let mut scanner = dataset.scan(); apply( &mut scanner, - r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v'"#, + r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#, ) .unwrap(); let mut swapped = Vec::new(); @@ -325,6 +414,8 @@ mod tests { assert_eq!( swapped, [ + ("json_array_contains".to_string(), false), + ("json_array_contains".to_string(), true), ("json_exists".to_string(), true), ("json_extract".to_string(), true), ("json_get_string".to_string(), false), diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index 887d7e3191..e8138e7e35 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -204,6 +204,25 @@ def test_filter_types(vectors, index): assert sorted(hit["key"] for hit in hits) == ["0", "3"] +def test_filter_arrays(vectors, index): + rows = {"space": ["space", "classic"], "nums": [1, 2.5], "strs": ["1", "2.5"], "flag": [True], "scalar": "space"} + vectors.put_vectors(**index, vectors=[{"key": key, "data": {"float32": [1, 0, 0, 0]}, "metadata": {"tags": tags}} + for key, tags in rows.items()]) + + def matches(condition): + hits = vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=10, + filter={"tags": condition})["vectors"] + return sorted(hit["key"] for hit in hits) + + assert matches("space") == ["scalar", "space"] + assert matches({"$eq": 1}) == ["nums"] + assert matches({"$eq": "1"}) == ["strs"] + assert matches({"$eq": True}) == ["flag"] + assert matches({"$in": ["classic", 2.5]}) == ["nums", "space"] + assert matches({"$ne": "space"}) == ["flag", "nums", "strs"] + assert matches({"$nin": [1, "2.5"]}) == ["flag", "scalar", "space"] + + def test_cosine_metric(vectors): bucket = "cosine-" + uuid.uuid4().hex[:16] vectors.create_vector_bucket(vectorBucketName=bucket) diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index d647668b18..77213ba2f9 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -1712,16 +1712,12 @@ func fieldClause(field string, cond interface{}) (string, error) { return "", fmt.Errorf("$exists expects a boolean") } case "$eq", "$ne": - lit, err := sqlLiteral(operand) - if err != nil { - return "", err - } - match, err := fieldMatch(field, operand, "= "+lit) + match, err := equalMatch(field, operand) if err != nil { return "", err } if op == "$ne" { - match = "NOT COALESCE(" + match + ", false)" + match = "NOT " + match } parts = append(parts, match) case "$in", "$nin": @@ -1761,6 +1757,13 @@ func fieldClause(field string, cond interface{}) (string, error) { for _, col := range cols { groups = append(groups, fmt.Sprintf("COALESCE(%s in (%s), false)", col, strings.Join(members[col], ", "))) } + for _, item := range list { + element, err := elementMatch(field, item) + if err != nil { + return "", err + } + groups = append(groups, element) + } expr := "(" + strings.Join(groups, " OR ") + ")" if op == "$nin" { expr = "NOT " + expr @@ -1789,14 +1792,41 @@ func fieldClause(field string, cond interface{}) (string, error) { } return strings.Join(parts, " AND "), nil default: - lit, err := sqlLiteral(cond) - if err != nil { - return "", err - } - return fieldMatch(field, cond, "= "+lit) + return equalMatch(field, cond) } } +// equalMatch renders "field equals operand": the stored value itself or, as +// AWS documents for $eq, any element of a stored array. +func equalMatch(field string, operand interface{}) (string, error) { + lit, err := sqlLiteral(operand) + if err != nil { + return "", err + } + match, err := fieldMatch(field, operand, "= "+lit) + if err != nil { + return "", err + } + element, err := elementMatch(field, operand) + if err != nil { + return "", err + } + return "(COALESCE(" + match + ", false) OR " + element + ")", nil +} + +// elementMatch renders "the array stored at field holds operand". The engine +// compares the elements as typed JSON values, so 1 does not match "1". +func elementMatch(field string, operand interface{}) (string, error) { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + if err := enc.Encode(operand); err != nil { + return "", err + } + return fmt.Sprintf("COALESCE(json_array_contains(`metadata`, '%s', '%s'), false)", + sqlEscape(existsPath(field)), sqlEscape(strings.TrimSuffix(b.String(), "\n"))), nil +} + // jsonbString renders s as the JSON text json_extract returns for a stored // string: quoted, with the same escapes the jsonb serializer emits. func jsonbString(s string) string { diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index 32dbef2d70..76849850f6 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -353,19 +353,26 @@ func TestResolveIndexARNs(t *testing.T) { } func TestTranslateFilter(t *testing.T) { + str := "json_extract(`metadata`, '$[\"k\"]')" + num := "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE)" + elem := func(value string) string { + return "COALESCE(json_array_contains(`metadata`, '$[\"k\"]', '" + value + "'), false)" + } cases := []struct { name string filter map[string]interface{} want string }{ {"bare string equality", map[string]interface{}{"k": "v"}, - "json_extract(`metadata`, '$[\"k\"]') = '\"v\"'"}, + "(COALESCE(" + str + " = '\"v\"', false) OR " + elem(`"v"`) + ")"}, {"bare number equality", map[string]interface{}{"k": float64(3)}, - "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) = 3"}, + "(COALESCE(" + num + " = 3, false) OR " + elem("3") + ")"}, {"bare bool equality", map[string]interface{}{"k": true}, - "json_extract(`metadata`, '$[\"k\"]') = 'true'"}, + "(COALESCE(" + str + " = 'true', false) OR " + elem("true") + ")"}, + {"eq fraction", map[string]interface{}{"k": map[string]interface{}{"$eq": float64(2.5)}}, + "(COALESCE(" + num + " = 2.5, false) OR " + elem("2.5") + ")"}, {"ne", map[string]interface{}{"k": map[string]interface{}{"$ne": "v"}}, - "NOT COALESCE(json_extract(`metadata`, '$[\"k\"]') = '\"v\"', false)"}, + "NOT (COALESCE(" + str + " = '\"v\"', false) OR " + elem(`"v"`) + ")"}, {"exists true", map[string]interface{}{"k": map[string]interface{}{"$exists": true}}, "json_exists(`metadata`, '$[\"k\"]')"}, {"exists false", map[string]interface{}{"k": map[string]interface{}{"$exists": false}}, @@ -373,19 +380,23 @@ func TestTranslateFilter(t *testing.T) { {"exists special key", map[string]interface{}{`a.b"c`: map[string]interface{}{"$exists": true}}, "json_exists(`metadata`, '$[\"a.b\\\"c\"]')"}, {"range", map[string]interface{}{"k": map[string]interface{}{"$gt": float64(1)}}, - "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) > 1"}, + num + " > 1"}, {"in strings", map[string]interface{}{"k": map[string]interface{}{"$in": []interface{}{"a", "b"}}}, - "(COALESCE(json_extract(`metadata`, '$[\"k\"]') in ('\"a\"', '\"b\"'), false))"}, + "(COALESCE(" + str + " in ('\"a\"', '\"b\"'), false) OR " + elem(`"a"`) + " OR " + elem(`"b"`) + ")"}, {"nin numbers", map[string]interface{}{"k": map[string]interface{}{"$nin": []interface{}{float64(1)}}}, - "NOT (COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) in (1), false))"}, + "NOT (COALESCE(" + num + " in (1), false) OR " + elem("1") + ")"}, {"in mixed types", map[string]interface{}{"k": map[string]interface{}{"$in": []interface{}{"a", float64(1), true, float64(2)}}}, - "(COALESCE(json_extract(`metadata`, '$[\"k\"]') in ('\"a\"', 'true'), false) OR COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) in (1, 2), false))"}, + "(COALESCE(" + str + " in ('\"a\"', 'true'), false) OR COALESCE(" + num + " in (1, 2), false) OR " + + elem(`"a"`) + " OR " + elem("1") + " OR " + elem("true") + " OR " + elem("2") + ")"}, {"and", map[string]interface{}{"$and": []interface{}{ map[string]interface{}{"a": "1"}, map[string]interface{}{"b": float64(2)}, }}, - "(json_extract(`metadata`, '$[\"a\"]') = '\"1\"' AND TRY_CAST(json_extract(`metadata`, '$[\"b\"]') AS DOUBLE) = 2)"}, + "((COALESCE(json_extract(`metadata`, '$[\"a\"]') = '\"1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"a\"]', '\"1\"'), false)) AND " + + "(COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"b\"]') AS DOUBLE) = 2, false) OR COALESCE(json_array_contains(`metadata`, '$[\"b\"]', '2'), false)))"}, {"sql injection", map[string]interface{}{"a'b": "v'1"}, - "json_extract(`metadata`, '$[\"a''b\"]') = '\"v''1\"'"}, + "(COALESCE(json_extract(`metadata`, '$[\"a''b\"]') = '\"v''1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"a''b\"]', '\"v''1\"'), false))"}, + {"array value escapes", map[string]interface{}{"k": ``}, + "(COALESCE(" + str + ` = '""', false) OR ` + elem(`""`) + ")"}, } for _, tc := range cases { got, err := translateFilter(tc.filter) From b9bc42c217280ad2a1c806312973f7730c697073 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:49:01 -0800 Subject: [PATCH 10/31] s3vectors: vend write credentials to index data only to the vector engine and Lance worker (#12316) * security: share the filer admin token check CheckGrpcAdminPeer verifies an admin Bearer token inline. Move the check into IsFilerAdminToken so an HTTP endpoint can accept the same token. Co-Authored-By: Claude Opus 5.5 * lance: recognize the cluster's own components by a signed token A request whose X-Seaweedfs-Cluster-Token header carries an admin token signed with jwt.filer_signing.key comes from a component that holds the cluster key: the vector engine or the Lance maintenance worker. Mark such a request as trusted. It still authenticates as an identity, which stays the principal for vended credentials. A dedicated header leaves the Authorization header to Lance OAuth clients. A gateway with no key trusts no one. Co-Authored-By: Claude Opus 5.5 * lance: vend write credentials to a vector bucket's indexes only to trusted callers An index in a vector bucket's vector namespace is also a Lance table, so a caller granted s3tables:UpdateTable could describe it, or declare it again, and get credentials that write the index data without passing the vector bucket policy. In that namespace both now vend read-only credentials unless the caller is trusted, and a declare from anyone else no longer writes the dataset markers. Co-Authored-By: Claude Opus 5.5 * lance worker: prove the cluster key to the Lance namespace The gateway now vends write credentials to a vector bucket's indexes only to callers that send a token signed with jwt.filer_signing.key. The worker already takes that key for the admin stream; its namespace client now mints a token from it for every catalog request. Co-Authored-By: Claude Opus 5.5 * weed-vector: describe the index the gateway created and prove the cluster key CreateIndex declared a table the gateway had just created, which only worked because declare is idempotent. Describe it instead. The Lance namespace vends write credentials to a vector bucket's indexes only to callers that send a token signed with jwt.filer_signing.key, so take the key with --filer-signing-key, the same flag the Lance worker has. Co-Authored-By: Claude Opus 5.5 * lance worker: keep the cluster key out of Debug output Co-Authored-By: Claude Opus 5.5 * lance: answer a declare of an existing table with its own location CreateTable succeeds for a name that already exists without looking at the location requested, and declare kept using that location for the dataset markers and the vended credentials. Declaring an existing table with another table's location, such as a vector index, got credentials to that table's data. Use the location the catalog holds, which CreateTable now also reports for a new table. Co-Authored-By: Claude Opus 5.5 * lance: sign cluster tokens with a key derived for the catalog A cluster token was a filer admin token signed with jwt.filer_signing.key and, by default, never expired. Sent to the Lance namespace on every request, often over plain HTTP, a captured one would also authorize the filer's IAM service. Sign it with a key derived from the cluster key for this one use, which the admin services do not accept, and mint it with a five minute lifetime. weed-vector drops its token lifetime flag. Co-Authored-By: Claude Opus 5.5 * lance: follow a reloaded jwt.filer_signing.key The Lance namespace copied the cluster key at startup. A SIGHUP reload refreshes the gateway's filer guard but not that copy, so after a key rotation the restarted vector engine and worker were no longer trusted. Read the key from the guard on every check. Co-Authored-By: Claude Opus 5.5 * iceberg: keep a vector bucket's indexes read-only An index in a vector bucket is also a catalog table the Iceberg REST catalog loads, and LoadTable vended credentials that could write it to any caller granted s3tables:UpdateTable. A commit staged its metadata file in the index directory on the same grant. Apply the vector namespace rule to both, after the caller's own UpdateTable check. Co-Authored-By: Claude Opus 5.5 * iceberg: confine a created table's location to its namespace before writing Stage-create skips the registration that confines a location to the table's namespace, yet vends write credentials for that location. A caller allowed to create a table in any namespace of a bucket could name another namespace's prefix, a vector index included, and get write credentials to it. The non-staged create also wrote its first metadata file there before the registration refused the location. Co-Authored-By: Claude Opus 5.5 * lance: a declare that finds an existing table answers like describe Creating a name that already exists is idempotent, so a caller allowed only to create tables could declare someone else's table and get write credentials to it, and clear its deregistered marker. Describe would have vended that caller read-only credentials. A declare now looks the name up first; when it only finds the table, writing needs UpdateTable, as it does for describe. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- seaweed-vector/crates/server/src/main.rs | 11 +- seaweed-vector/crates/server/src/service.rs | 3 +- seaweed-worker/README.md | 5 + seaweed-worker/crates/core/src/token.rs | 19 ++- .../crates/lance/src/catalog/mod.rs | 2 +- .../crates/lance/src/catalog/namespace.rs | 121 +++++++++++++++- .../crates/lance/src/jobs/cleanup.rs | 13 +- .../crates/lance/src/jobs/compact.rs | 13 +- .../crates/lance/src/jobs/indices.rs | 13 +- seaweed-worker/crates/lance/src/jobs/mod.rs | 5 + seaweed-worker/crates/lance/src/jobs/sort.rs | 13 +- seaweed-worker/crates/lance/src/main.rs | 9 +- seaweed-worker/crates/lance/src/preview.rs | 13 +- weed/command/s3.go | 1 + weed/s3api/iceberg/commit_helpers.go | 4 + weed/s3api/iceberg/handlers_table.go | 9 ++ weed/s3api/iceberg/vector_namespace_test.go | 130 ++++++++++++++++++ weed/s3api/lance/cluster_token_test.go | 94 +++++++++++++ weed/s3api/lance/handlers_table.go | 49 +++++-- weed/s3api/lance/handlers_test.go | 43 ++++++ weed/s3api/lance/server.go | 57 ++++++++ weed/s3api/lance/storage.go | 17 +++ weed/s3api/lance/vector_namespace_test.go | 88 ++++++++++++ weed/s3api/s3api_server.go | 9 ++ weed/s3api/s3tables/handler_table.go | 5 +- weed/security/grpc_peer.go | 20 ++- 26 files changed, 724 insertions(+), 42 deletions(-) create mode 100644 weed/s3api/iceberg/vector_namespace_test.go create mode 100644 weed/s3api/lance/cluster_token_test.go diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index 10d46f372d..aa42defcc7 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Parser; use tonic::transport::Server; -use weed_lance_worker::catalog::NamespaceClient; +use weed_lance_worker::catalog::{NamespaceClient, SigningKey}; use weed_lance_worker::dataset::FallbackOptions; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; use weed_vector_server::read_cache::ReadCache; @@ -31,6 +31,10 @@ struct Args { /// Memory for caching reads of write-once table files, in MiB; 0 disables. #[arg(long, env = "WEED_VECTOR_READ_CACHE_MB", default_value_t = 512)] read_cache_mb: u64, + /// The `jwt.filer_signing.key` of the cluster. The Lance namespace vends + /// credentials that write index data only to a caller holding it. + #[arg(long, env = "WEED_FILER_SIGNING_KEY")] + filer_signing_key: Option, } fn parse_option(s: &str) -> Result<(String, String), String> { @@ -111,7 +115,10 @@ async fn main() -> Result<()> { )); } let fallback: FallbackOptions = args.storage_options.into_iter().collect(); - let client = NamespaceClient::new(args.namespace.clone()).with_fallback(&fallback); + let signing_key = SigningKey::new(args.filer_signing_key); + let client = NamespaceClient::new(args.namespace.clone()) + .with_fallback(&fallback) + .with_signing_key(signing_key.as_ref()); let mut service = VectorServer::new(client, fallback); if args.read_cache_mb > 0 { service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20)); diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index cb9f51b1c4..ea26f6c9bf 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -424,9 +424,10 @@ impl VectorService for VectorServer { if dimension == 0 || dimension > 4096 { return Err(Status::invalid_argument("dimension must be 1..=4096")); } + // The gateway has already created the table through the catalog. let description = self .client - .declare_table(&table.id, None) + .describe_table(&table.id) .await .map_err(internal)?; let mut options = self.fallback.clone(); diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index 9dde317711..02497807d9 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -95,6 +95,11 @@ gateway without STS configured vends no credentials at all, so `--access-key` and `--secret-key` supply a fallback; anything the namespace does vend wins over them. +In a vector bucket's `vector` namespace the gateway vends write credentials only +to a caller that proves it holds `jwt.filer_signing.key`. Pass that key with +`--filer-signing-key` so compaction, cleanup and index builds can rewrite +vector indexes. + ## State All three jobs are implemented and tested end to end against a live gateway: diff --git a/seaweed-worker/crates/core/src/token.rs b/seaweed-worker/crates/core/src/token.rs index 3df69398ef..21c01b9e08 100644 --- a/seaweed-worker/crates/core/src/token.rs +++ b/seaweed-worker/crates/core/src/token.rs @@ -33,9 +33,26 @@ pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option Vec { + let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key); + hmac::sign(&key, purpose.as_bytes()).as_ref().to_vec() +} + #[cfg(test)] mod tests { - use super::admin_token; + use super::{admin_token, derive_key}; + + #[test] + fn derive_key_matches_the_go_derivation() { + let derived = derive_key(b"cluster-key", "seaweedfs-lance-cluster-token"); + let hex: String = derived.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!( + hex, + "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021" + ); + } #[test] fn no_key_no_token() { diff --git a/seaweed-worker/crates/lance/src/catalog/mod.rs b/seaweed-worker/crates/lance/src/catalog/mod.rs index 142de4eb52..a3a91ad307 100644 --- a/seaweed-worker/crates/lance/src/catalog/mod.rs +++ b/seaweed-worker/crates/lance/src/catalog/mod.rs @@ -7,4 +7,4 @@ pub mod namespace; -pub use namespace::{NamespaceClient, TableDescription, parse_id}; +pub use namespace::{NamespaceClient, SigningKey, TableDescription, parse_id}; diff --git a/seaweed-worker/crates/lance/src/catalog/namespace.rs b/seaweed-worker/crates/lance/src/catalog/namespace.rs index 6aca480495..bbbfda6f0f 100644 --- a/seaweed-worker/crates/lance/src/catalog/namespace.rs +++ b/seaweed-worker/crates/lance/src/catalog/namespace.rs @@ -8,6 +8,39 @@ use serde::Deserialize; /// The delimiter the Lance namespace joins identifier parts with. const DELIMITER: &str = "$"; +/// Carries a token that proves the caller holds the cluster key, which marks +/// it as one of the cluster's own components. +const CLUSTER_TOKEN_HEADER: &str = "X-Seaweedfs-Cluster-Token"; + +/// A cluster token is minted for each request, so it only has to outlast the +/// clock skew between this host and the gateway. +const CLUSTER_TOKEN_SECS: u64 = 300; + +/// The key cluster tokens are signed with, derived from the cluster key +/// (`jwt.filer_signing.key`) the way the gateway derives it, so a token +/// captured on the wire does not authorize admin calls. The catalog vends +/// credentials that write a vector bucket's index data only to callers that +/// send one. Not Debug, so the key cannot reach a log. +#[derive(Clone)] +pub struct SigningKey { + key: Vec, +} + +impl SigningKey { + pub fn new(key: Option) -> Option { + key.filter(|key| !key.is_empty()).map(|key| Self { + key: seaweed_worker_core::token::derive_key( + key.as_bytes(), + "seaweedfs-lance-cluster-token", + ), + }) + } + + fn token(&self) -> Option { + seaweed_worker_core::token::admin_token(&self.key, CLUSTER_TOKEN_SECS) + } +} + #[derive(Debug, Deserialize)] pub struct TableDescription { pub location: String, @@ -35,6 +68,7 @@ pub struct NamespaceClient { base_url: String, http: reqwest::Client, api_key: Option, + signing_key: Option, } /// A namespace call that has not answered by now is not going to. Without this @@ -71,6 +105,7 @@ impl NamespaceClient { base_url: base_url.into().trim_end_matches('/').to_string(), http, api_key: None, + signing_key: None, } } @@ -92,10 +127,25 @@ impl NamespaceClient { self } + pub fn with_signing_key(mut self, signing_key: Option<&SigningKey>) -> Self { + if signing_key.is_some() && !protected_transport(&self.base_url) { + tracing::warn!( + "catalog url {} exposes the cluster token in cleartext", + self.base_url + ); + } + self.signing_key = signing_key.cloned(); + self + } + fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - match &self.api_key { + let request = match &self.api_key { Some(key) => request.header("x-api-key", key), None => request, + }; + match self.signing_key.as_ref().and_then(SigningKey::token) { + Some(token) => request.header(CLUSTER_TOKEN_HEADER, token), + None => request, } } @@ -224,7 +274,74 @@ pub fn parse_id(encoded: &str) -> Vec { #[cfg(test)] mod tests { - use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin}; + use super::{ + CLUSTER_TOKEN_SECS, MAX_REDIRECT_HOPS, NamespaceClient, SigningKey, may_follow, + protected_transport, same_origin, + }; + + /// A catalog that answers one request with a table location and hands back + /// the request it read. + async fn recording_catalog() -> (String, tokio::sync::oneshot::Receiver) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (sender, receiver) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let (mut conn, _) = listener.accept().await.unwrap(); + let mut request = vec![0u8; 8192]; + let n = conn.read(&mut request).await.unwrap(); + let _ = sender.send(String::from_utf8_lossy(&request[..n]).into_owned()); + let body = r#"{"location":"s3://b/vector/t"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = conn.write_all(response.as_bytes()).await; + }); + (url, receiver) + } + + fn header<'a>(request: &'a str, name: &str) -> Option<&'a str> { + request.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case(name).then(|| value.trim()) + }) + } + + #[tokio::test] + async fn signing_key_sends_a_cluster_token() { + let id = ["b".to_string(), "vector".to_string(), "t".to_string()]; + let (url, request) = recording_catalog().await; + let key = SigningKey::new(Some("cluster-key".into())); + NamespaceClient::new(url) + .with_signing_key(key.as_ref()) + .describe_table(&id) + .await + .unwrap(); + let request = request.await.unwrap(); + let token = header(&request, "x-seaweedfs-cluster-token").expect("cluster token"); + let raw = + seaweed_worker_core::token::admin_token(b"cluster-key", CLUSTER_TOKEN_SECS).unwrap(); + assert_eq!(token.split('.').count(), 3); + assert_ne!( + token.rsplit('.').next(), + raw.rsplit('.').next(), + "signed with the cluster key itself" + ); + + let (url, request) = recording_catalog().await; + NamespaceClient::new(url).describe_table(&id).await.unwrap(); + assert_eq!( + header(&request.await.unwrap(), "x-seaweedfs-cluster-token"), + None + ); + } + + #[test] + fn an_empty_signing_key_is_no_key() { + assert!(SigningKey::new(None).is_none()); + assert!(SigningKey::new(Some(String::new())).is_none()); + } #[test] fn may_follow_bounds_hops_within_the_origin() { diff --git a/seaweed-worker/crates/lance/src/jobs/cleanup.rs b/seaweed-worker/crates/lance/src/jobs/cleanup.rs index c8c4a3ec36..028739d688 100644 --- a/seaweed-worker/crates/lance/src/jobs/cleanup.rs +++ b/seaweed-worker/crates/lance/src/jobs/cleanup.rs @@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{clamp, string_list, table_id}; @@ -32,6 +32,7 @@ const MAX_MIN_VERSIONS: i64 = 1000; pub struct CleanupVersionsHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -40,6 +41,7 @@ impl CleanupVersionsHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -54,8 +56,15 @@ impl CleanupVersionsHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/compact.rs b/seaweed-worker/crates/lance/src/jobs/compact.rs index 728437420d..a4ae5d7565 100644 --- a/seaweed-worker/crates/lance/src/jobs/compact.rs +++ b/seaweed-worker/crates/lance/src/jobs/compact.rs @@ -12,7 +12,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{FORMAT, clamp, observation, string_list, table_id}; @@ -35,6 +35,7 @@ const MIN_FRAGMENTS_CEILING: i64 = 4096; pub struct CompactHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -43,6 +44,7 @@ impl CompactHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -58,8 +60,15 @@ impl CompactHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/indices.rs b/seaweed-worker/crates/lance/src/jobs/indices.rs index e619ff43d5..24f87b0c68 100644 --- a/seaweed-worker/crates/lance/src/jobs/indices.rs +++ b/seaweed-worker/crates/lance/src/jobs/indices.rs @@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset::{self, OpenTable}; use crate::jobs::{clamp, string_list, table_id}; @@ -35,6 +35,7 @@ const MAX_UNINDEXED_CEILING: i64 = 100_000_000; pub struct OptimizeIndicesHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -43,6 +44,7 @@ impl OptimizeIndicesHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -57,8 +59,15 @@ impl OptimizeIndicesHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/mod.rs b/seaweed-worker/crates/lance/src/jobs/mod.rs index b5502a7747..a9c3bcfb51 100644 --- a/seaweed-worker/crates/lance/src/jobs/mod.rs +++ b/seaweed-worker/crates/lance/src/jobs/mod.rs @@ -42,27 +42,32 @@ pub(crate) fn table_id(parameters: &HashMap) -> Option, metrics: Option, ) -> Vec> { vec![ Arc::new( compact::CompactHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( indices::OptimizeIndicesHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( sort::SortHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( cleanup::CleanupVersionsHandler::new(namespace_url) .with_fallback(fallback) + .with_signing_key(signing_key) .with_metrics(metrics), ), ] diff --git a/seaweed-worker/crates/lance/src/jobs/sort.rs b/seaweed-worker/crates/lance/src/jobs/sort.rs index ed8aeaa7ba..5ac6859f6a 100644 --- a/seaweed-worker/crates/lance/src/jobs/sort.rs +++ b/seaweed-worker/crates/lance/src/jobs/sort.rs @@ -23,7 +23,7 @@ use seaweed_worker_sort::{ }; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{FORMAT, clamp, observation, string_list, table_id}; @@ -54,6 +54,7 @@ const BYTES_PER_MB: u64 = 1024 * 1024; pub struct SortHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -62,6 +63,7 @@ impl SortHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -77,8 +79,15 @@ impl SortHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 8f688cf357..b47a20bf8a 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -63,7 +63,8 @@ struct Args { /// Shared secret for the admin control stream, the `jwt.filer_signing.key` /// value the Go worker reads from its config file. Required by admins that - /// authenticate the worker stream with a signing key. + /// authenticate the worker stream with a signing key. The Lance namespace + /// takes it too: only a holder gets write credentials to vector indexes. #[arg(long, env = "WEED_FILER_SIGNING_KEY")] filer_signing_key: Option, @@ -136,6 +137,7 @@ async fn main() -> Result<()> { let args = Args::parse(); let tls = args.tls()?; + let signing_key = weed_lance_worker::catalog::SigningKey::new(args.filer_signing_key.clone()); let options = WorkerOptions { admin_address: args.admin, worker_id: args.id, @@ -164,9 +166,10 @@ async fn main() -> Result<()> { } let mut registry = Registry::new().with_preview(Arc::new( - weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()), + weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()) + .with_signing_key(signing_key.clone()), )); - for handler in handlers(args.namespace, fallback, Some(lance_metrics)) { + for handler in handlers(args.namespace, fallback, signing_key, Some(lance_metrics)) { registry = registry.register(handler); } diff --git a/seaweed-worker/crates/lance/src/preview.rs b/seaweed-worker/crates/lance/src/preview.rs index 596d8700a0..49dd6e2775 100644 --- a/seaweed-worker/crates/lance/src/preview.rs +++ b/seaweed-worker/crates/lance/src/preview.rs @@ -10,13 +10,14 @@ use async_trait::async_trait; use futures::TryStreamExt; use seaweed_worker_core::{Preview, PreviewProvider}; -use crate::catalog::NamespaceClient; +use crate::catalog::{NamespaceClient, SigningKey}; use crate::dataset::{self, FallbackOptions}; use crate::jobs::FORMAT; pub struct LancePreview { namespace_url: String, fallback: FallbackOptions, + signing_key: Option, } impl LancePreview { @@ -24,8 +25,14 @@ impl LancePreview { Self { namespace_url, fallback, + signing_key: None, } } + + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } } #[async_trait] @@ -35,7 +42,9 @@ impl PreviewProvider for LancePreview { } async fn preview(&self, object_id: &[String], row_limit: usize) -> Result { - let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback); + let client = NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()); let table = dataset::open(&client, object_id, &self.fallback).await?; let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64; diff --git a/weed/command/s3.go b/weed/command/s3.go index f74bcda38b..94ed45d9f0 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -675,6 +675,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenO lanceServer := lance.NewServer(s3ApiServer, s3ApiServer) lanceServer.SetCredentialValidator(s3ApiServer) + lanceServer.SetSigningKey(s3ApiServer.FilerSigningKey) if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" { lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer}) } diff --git a/weed/s3api/iceberg/commit_helpers.go b/weed/s3api/iceberg/commit_helpers.go index bd725d019d..8dd19818ec 100644 --- a/weed/s3api/iceberg/commit_helpers.go +++ b/weed/s3api/iceberg/commit_helpers.go @@ -282,6 +282,10 @@ func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespac Name: tableName, }, identityName) }) + if err == nil { + bucketName, _ := s3tables.ParseBucketNameFromARN(bucketARN) + err = s.tablesManager.CheckVectorNamespaceWrite(ctx, s.filerClient, bucketName, namespace) + } if err == nil { return nil } diff --git a/weed/s3api/iceberg/handlers_table.go b/weed/s3api/iceberg/handlers_table.go index d99ba5a708..98debaeb12 100644 --- a/weed/s3api/iceberg/handlers_table.go +++ b/weed/s3api/iceberg/handlers_table.go @@ -308,6 +308,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) { writeManagerError(w, authErr) return } + // Registration confines the location to the table's namespace, but both + // branches write, and stage-create vends credentials, before it runs. + if err := s3tables.ValidateMetadataLocation(metadataLocation, bucketName, flattenNamespacePath(namespace)); err != nil { + writeError(w, http.StatusBadRequest, "BadRequestException", err.Error()) + return + } // Stage-create persists metadata in the internal staged area and skips S3Tables registration. if req.StageCreate { @@ -786,6 +792,9 @@ func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace [] err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName) }) + if err == nil { + err = s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucketName, namespace) + } if err != nil { glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err) } diff --git a/weed/s3api/iceberg/vector_namespace_test.go b/weed/s3api/iceberg/vector_namespace_test.go new file mode 100644 index 0000000000..dad8e4ecb6 --- /dev/null +++ b/weed/s3api/iceberg/vector_namespace_test.go @@ -0,0 +1,130 @@ +package iceberg + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +type openAuthenticator struct{} + +func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) { + return s3tables.DefaultAccountID, nil, s3err.ErrNone +} + +func (openAuthenticator) DefaultAllow() bool { return true } + +// An index in a vector bucket is a catalog table the Iceberg catalog can load, +// but a caller holding every s3tables grant gets neither write credentials to +// it nor a commit into it. +func TestVectorIndexIsReadOnlyOverIceberg(t *testing.T) { + filer := s3tablestest.Start(t) + vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}} + s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{}) + s.SetCredentialVendor(vendor) + s.s3Endpoint = "http://127.0.0.1:8333" + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + + admin := s3tables.NewManager() + admin.SetTrusted(true) + client := s3tables.NewManagerClient(filer.Client) + arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec") + if err != nil { + t.Fatal(err) + } + metadata, _ := json.Marshal(map[string]any{"dimension": 4, "distanceMetric": "cosine"}) + for _, op := range []struct { + name string + req interface{} + }{ + {"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}}, + {"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, + Name: "idx", Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadata}}}, + } { + if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil { + t.Fatalf("%s: %v", op.name, err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/v1/vec/namespaces/vector/tables/idx", nil) + req.Header.Set(accessDelegationHeader, "vended-credentials") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("load = %d (%s)", rec.Code, rec.Body.String()) + } + if vendor.gotPrefix != "vector/idx" || !vendor.gotReadOnly { + t.Errorf("vended prefix %q read-only %v, want vector/idx read-only", vendor.gotPrefix, vendor.gotReadOnly) + } + + reqErr := s.authorizeCommit(t.Context(), arn, []string{s3tables.VectorIndexNamespace}, "idx", s3tables.DefaultAccountID) + if reqErr == nil || reqErr.status != http.StatusForbidden { + t.Errorf("commit authorization = %+v, want 403", reqErr) + } +} + +// Stage-create registers nothing, so the catalog's own location and vector +// namespace checks never see it. It must not hand out write credentials to an +// index, whether the staged table names the index's location from another +// namespace or sits in the vector namespace itself. +func TestStageCreateCannotReachVectorIndex(t *testing.T) { + filer := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{}) + s.s3Endpoint = "http://127.0.0.1:8333" + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + + admin := s3tables.NewManager() + admin.SetTrusted(true) + client := s3tables.NewManagerClient(filer.Client) + arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec") + if err != nil { + t.Fatal(err) + } + for _, op := range []struct { + name string + req interface{} + }{ + {"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{"other"}}}, + {"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, + Name: "idx", Format: s3tables.FormatLance}}, + } { + if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil { + t.Fatalf("%s: %v", op.name, err) + } + } + + for name, tc := range map[string]struct { + namespace, body string + status int + }{ + "index location from another namespace": {"other", `{"name":"x","stage-create":true,"location":"s3://vec/vector/idx"}`, http.StatusBadRequest}, + "vector namespace": {s3tables.VectorIndexNamespace, `{"name":"y","stage-create":true}`, http.StatusForbidden}, + } { + t.Run(name, func(t *testing.T) { + vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}} + s.SetCredentialVendor(vendor) + req := httptest.NewRequest(http.MethodPost, "/v1/vec/namespaces/"+tc.namespace+"/tables", strings.NewReader(tc.body)) + req.Header.Set(accessDelegationHeader, "vended-credentials") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != tc.status { + t.Errorf("stage-create = %d (%s), want %d", rec.Code, rec.Body.String(), tc.status) + } + if vendor.gotPrefix != "" && !vendor.gotReadOnly { + t.Errorf("vended write credentials to %q", vendor.gotPrefix) + } + }) + } +} diff --git a/weed/s3api/lance/cluster_token_test.go b/weed/s3api/lance/cluster_token_test.go new file mode 100644 index 0000000000..15409a3aa3 --- /dev/null +++ b/weed/s3api/lance/cluster_token_test.go @@ -0,0 +1,94 @@ +package lance + +import ( + "encoding/hex" + "net/http" + "net/http/httptest" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/security" +) + +func signAdminClaims(t *testing.T, key security.SigningKey, claims security.SeaweedFilerAdminClaims) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(key)) + if err != nil { + t.Fatalf("sign: %v", err) + } + return token +} + +func trustedSeenByHandler(t *testing.T, s *Server, token string) bool { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + if token != "" { + req.Header.Set(clusterTokenHeader, token) + } + trusted := false + rec := httptest.NewRecorder() + s.Auth(func(w http.ResponseWriter, r *http.Request) { + trusted = isTrustedCaller(r.Context()) + w.WriteHeader(http.StatusOK) + })(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d (%s), want 200", rec.Code, rec.Body.String()) + } + return trusted +} + +func TestClusterTokenMarksTheCallerTrusted(t *testing.T) { + key := security.SigningKey("cluster-key") + s := NewServer(nil, openAuthenticator{}) + s.SetSigningKey(func() security.SigningKey { return key }) + + derived := clusterTokenKey(key) + expired := security.SeaweedFilerAdminClaims{} + expired.ExpiresAt = jwt.NewNumericDate(time.Now().Add(-time.Minute)) + for _, tc := range []struct { + name string + token string + want bool + }{ + {"no token", "", false}, + {"signed with the derived key", string(security.GenJwtForFilerAdmin(derived, 60)), true}, + {"signed with the cluster key itself", string(security.GenJwtForFilerAdmin(key, 60)), false}, + {"signed with another key", string(security.GenJwtForFilerAdmin(clusterTokenKey(security.SigningKey("other-key")), 60)), false}, + {"expired", signAdminClaims(t, derived, expired), false}, + {"session token", signAdminClaims(t, derived, security.SeaweedFilerAdminClaims{SessionId: "session"}), false}, + {"garbage", "not-a-token", false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := trustedSeenByHandler(t, s, tc.token); got != tc.want { + t.Errorf("trusted = %v, want %v", got, tc.want) + } + }) + } + + token := string(security.GenJwtForFilerAdmin(derived, 60)) + key = security.SigningKey("rotated-key") + if trustedSeenByHandler(t, s, token) { + t.Error("a token from before a key rotation is still trusted") + } + if !trustedSeenByHandler(t, s, string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))) { + t.Error("a token signed after a key rotation is not trusted") + } + + key = nil + if trustedSeenByHandler(t, s, token) { + t.Error("a server with no cluster key trusted a caller") + } + s.SetSigningKey(nil) + if trustedSeenByHandler(t, s, token) { + t.Error("a server with no key source trusted a caller") + } +} + +// The Rust catalog client derives the same key; its tests pin this value too. +func TestClusterTokenKeyDerivation(t *testing.T) { + got := hex.EncodeToString(clusterTokenKey(security.SigningKey("cluster-key"))) + if want := "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"; got != want { + t.Errorf("clusterTokenKey = %s, want %s", got, want) + } +} diff --git a/weed/s3api/lance/handlers_table.go b/weed/s3api/lance/handlers_table.go index 65c0e1a11d..c2027d2d32 100644 --- a/weed/s3api/lance/handlers_table.go +++ b/weed/s3api/lance/handlers_table.go @@ -135,24 +135,35 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) { return } - if err := s.createTable(r, bucket, ns, name, location); err != nil { + _, lookupErr := s.loadLanceTable(r, bucket, ns, name) + location, err = s.createTable(r, bucket, ns, name, location) + if err != nil { writeStorageError(w, r, err) return } - if err := s.writeMarker(r, location, reservedMarker); err != nil { - // The catalog entry is the authority; the marker only mirrors it for - // clients that read the storage prefix directly. - glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err) + // A declare that creates the table may write it: a Lance commit is a direct + // S3 write that never revisits the catalog, so the create authorization is + // the write evidence. One that only finds the table answers like describe. + var writable bool + if isNotFound(lookupErr) { + writable = s.mayWriteNamespace(r, bucket, ns) + } else { + writable = s.mayUpdateTable(r, bucket, ns, name) } - // Declaring a name that was deregistered brings it back, the same way - // registering it does. - if err := s.removeMarker(r, location, deregisteredMarker); err != nil { - glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err) + if writable { + if err := s.writeMarker(r, location, reservedMarker); err != nil { + // The catalog entry is the authority; the marker only mirrors it for + // clients that read the storage prefix directly. + glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err) + } + // Declaring a name that was deregistered brings it back, the same way + // registering it does. + if err := s.removeMarker(r, location, deregisteredMarker); err != nil { + glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err) + } } - // A Lance commit is a direct S3 write that never revisits the catalog, so - // the create authorization above is the write evidence for the dataset. - options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true }) + options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return writable }) if err != nil { writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error()) return @@ -353,7 +364,7 @@ func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) { } } default: - if err := s.createTable(r, bucket, ns, name, location); err != nil { + if location, err = s.createTable(r, bucket, ns, name, location); err != nil { writeStorageError(w, r, err) return } @@ -545,7 +556,9 @@ func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, nam return &lanceTable{location: location, versionToken: resp.VersionToken}, nil } -func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error { +// createTable returns the location the catalog holds for the table. When the +// name already exists that is its own location, not the one asked for. +func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) (string, error) { req := &s3tables.CreateTableRequest{ TableBucketARN: bucketARN(bucket), Namespace: ns, @@ -556,7 +569,13 @@ func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, MetadataLocation: location, } var resp s3tables.CreateTableResponse - return s.execute(r, "CreateTable", req, &resp) + if err := s.execute(r, "CreateTable", req, &resp); err != nil { + return "", err + } + if resp.MetadataLocation == "" { + return tableLocation(bucket, ns, name), nil + } + return resp.MetadataLocation, nil } // repointTable moves an existing entry to another dataset location without diff --git a/weed/s3api/lance/handlers_test.go b/weed/s3api/lance/handlers_test.go index d62bdc7089..72d9b84f68 100644 --- a/weed/s3api/lance/handlers_test.go +++ b/weed/s3api/lance/handlers_test.go @@ -415,3 +415,46 @@ func TestDataPlaneIsUnsupported(t *testing.T) { t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported) } } + +// createOnlyAuthenticator signs in an identity that may create and read tables +// but not update them. +type createOnlyAuthenticator struct{} + +func (createOnlyAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) { + return "creator", &struct { + Name string + Actions []string + }{Name: "creator", Actions: []string{"s3tables:CreateTable", "s3tables:GetTable"}}, s3err.ErrNone +} + +func (createOnlyAuthenticator) DefaultAllow() bool { return false } + +// The right to create a name is the write evidence for a table the declare +// creates, but not for one it only finds. +func TestDeclareVendsWriteOnlyForATableItCreates(t *testing.T) { + h := newTestHarness(t) + h.createBucket(t, "lnc", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/namespace/lnc$ns/create", `{}`, http.StatusOK) + h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/declare", `{}`, http.StatusOK) + h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/deregister", `{}`, http.StatusOK) + + server := NewServer(s3tables.NewManagerClient(h.filer.Client), createOnlyAuthenticator{}) + server.SetCredentialVendor(recordingVendor{}) + router := mux.NewRouter().SkipClean(true) + server.RegisterRoutes(router) + creator := &testHarness{router: router, filer: h.filer, admin: h.admin, server: server} + + for _, tc := range []struct{ name, want string }{ + {"theirs", "read-only"}, + {"mine", "read-write"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := creator.vendedKey(t, "/v1/table/lnc$ns$"+tc.name+"/declare", ""); got != tc.want { + t.Errorf("vended %q, want %q", got, tc.want) + } + }) + } + if h.filer.Get("/buckets/lnc/ns/theirs", deregisteredMarker) == nil { + t.Error("a declare that may not write the table brought it back") + } +} diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index 53effee9e5..5d25e8c860 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -2,6 +2,8 @@ package lance import ( "context" + "crypto/hmac" + "crypto/sha256" "net/http" "strings" "time" @@ -12,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/security" ) // FilerClient provides access to the filer for storage operations. @@ -61,6 +64,7 @@ type Server struct { credentialVendor CredentialVendor s3Endpoint string s3Region string + signingKey func() security.SigningKey } // NewServer creates a Lance namespace server over the given filer. @@ -89,6 +93,13 @@ func (s *Server) SetCredentialVendor(vendor CredentialVendor) { s.credentialVendor = vendor } +// SetSigningKey sets where to read the cluster key (jwt.filer_signing.key) +// that a trusted caller's clusterTokenHeader is checked against. It is read on +// every check, so a reloaded key takes effect at once. +func (s *Server) SetSigningKey(signingKey func() security.SigningKey) { + s.signingKey = signingKey +} + // SetS3Endpoint configures the S3 endpoint advertised in storage_options so a // client can reach the dataset without separately discovering the S3 address. func (s *Server) SetS3Endpoint(endpoint string) { @@ -171,6 +182,9 @@ func loggingMiddleware(next http.Handler) http.Handler { // header; the S3 authenticator stays for callers that can SigV4-sign. func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if s.hasClusterToken(r) { + r = r.WithContext(withTrustedCaller(r.Context())) + } // A request carrying a Bearer token is a Lance REST client. An invalid // or expired token gets 401 immediately; falling through to the S3 // authenticator would parse the header as SigV4 and fail with a @@ -250,3 +264,46 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { handler(w, r) } } + +// clusterTokenHeader carries a token signed with clusterTokenKey. The vector +// engine and the Lance maintenance worker send it next to their identity's +// credentials, and only they may write index data in a vector bucket. +const clusterTokenHeader = "X-Seaweedfs-Cluster-Token" + +// clusterTokenKey derives the key cluster tokens are signed with from the +// cluster key, so a token captured on its way here does not authorize the +// admin calls that take the cluster key itself. +func clusterTokenKey(key security.SigningKey) security.SigningKey { + mac := hmac.New(sha256.New, key) + mac.Write([]byte("seaweedfs-lance-cluster-token")) + return mac.Sum(nil) +} + +type trustedCallerKey struct{} + +func withTrustedCaller(ctx context.Context) context.Context { + return context.WithValue(ctx, trustedCallerKey{}, true) +} + +func isTrustedCaller(ctx context.Context) bool { + trusted, _ := ctx.Value(trustedCallerKey{}).(bool) + return trusted +} + +// hasClusterToken reports whether the request proves it holds the cluster key. +// Without a configured key no caller can. +func (s *Server) hasClusterToken(r *http.Request) bool { + token := r.Header.Get(clusterTokenHeader) + if token == "" || s.signingKey == nil { + return false + } + key := s.signingKey() + if len(key) == 0 { + return false + } + if security.IsFilerAdminToken(clusterTokenKey(key), security.EncodedJwt(token)) { + return true + } + glog.V(1).Infof("lance: %s from %s is not signed with a key derived from jwt.filer_signing.key", clusterTokenHeader, r.RemoteAddr) + return false +} diff --git a/weed/s3api/lance/storage.go b/weed/s3api/lance/storage.go index c4ccd67385..52a35640b5 100644 --- a/weed/s3api/lance/storage.go +++ b/weed/s3api/lance/storage.go @@ -177,6 +177,9 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b // credentials would cover, the same gate the Iceberg catalog applies so a // read-only caller never gets write-capable storage_options. func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool { + if !s.mayWriteNamespace(r, bucket, ns) { + return false + } req := &s3tables.UpdateTableRequest{ TableBucketARN: bucketARN(bucket), Namespace: ns, @@ -192,6 +195,20 @@ func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, nam return err == nil } +// mayWriteNamespace keeps the vector namespace of a vector bucket out of reach +// of s3tables grants: its index data is written only by trusted callers, and +// the vector bucket policy gates everyone else through S3 Vectors. +func (s *Server) mayWriteNamespace(r *http.Request, bucket string, ns []string) bool { + if isTrustedCaller(r.Context()) { + return true + } + err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns) + if err != nil { + glog.V(1).Infof("lance: %s/%s is read-only to this caller: %v", bucket, strings.Join(ns, "."), err) + } + return err == nil +} + // locationPrefix strips the s3://bucket/ part of a location, leaving the key // prefix a credential is scoped to. func locationPrefix(location string) string { diff --git a/weed/s3api/lance/vector_namespace_test.go b/weed/s3api/lance/vector_namespace_test.go index 01ec1770a8..3185684df3 100644 --- a/weed/s3api/lance/vector_namespace_test.go +++ b/weed/s3api/lance/vector_namespace_test.go @@ -1,10 +1,14 @@ package lance import ( + "context" "net/http" + "net/http/httptest" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/security" ) // seedVectorIndex creates bucket/vector/idx through the trusted path, the way @@ -75,3 +79,87 @@ func TestVectorNamespaceOfUndeclaredBucketIsWritableOverLance(t *testing.T) { h.mustDo(t, http.MethodPost, idx+"/register", `{"location":"s3://old/vector/idx"}`, http.StatusOK) h.mustDo(t, http.MethodPost, idx+"/drop", `{}`, http.StatusOK) } + +// recordingVendor vends a credential whose key says whether it may write and +// whose secret is the prefix it covers. +type recordingVendor struct{} + +func (recordingVendor) VendTableCredentials(_ context.Context, _, _, prefix string, readOnly bool) (*VendedCredentials, error) { + if readOnly { + return &VendedCredentials{AccessKeyID: "read-only", SecretAccessKey: prefix}, nil + } + return &VendedCredentials{AccessKeyID: "read-write", SecretAccessKey: prefix}, nil +} + +func (h *testHarness) vendedKey(t *testing.T, target, token string) string { + t.Helper() + req := httptest.NewRequest(http.MethodPost, target, strings.NewReader(`{"vend_credentials":true}`)) + if token != "" { + req.Header.Set(clusterTokenHeader, token) + } + rec := httptest.NewRecorder() + h.router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("POST %s = %d (%s)", target, rec.Code, rec.Body.String()) + } + return decode[DescribeTableResponse](t, rec).StorageOptions["aws_access_key_id"] +} + +// A caller holding every s3tables grant still gets read-only credentials to a +// vector bucket's index data; only the cluster's own components can write it. +func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) { + h := newTestHarness(t) + key := security.SigningKey("cluster-key") + h.server.SetSigningKey(func() security.SigningKey { return key }) + h.server.SetCredentialVendor(recordingVendor{}) + h.seedVectorIndex(t, "vec", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK) + h.seedVectorIndex(t, "old", "") + + h.vendedKey(t, "/v1/table/vec$vector$idx/declare", "") + if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil { + t.Error("an untrusted declare wrote into the index") + } + + trusted := string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60)) + forged := string(security.GenJwtForFilerAdmin(key, 60)) + for _, tc := range []struct { + name, target, token, want string + }{ + {"describe the index", "/v1/table/vec$vector$idx/describe", "", "read-only"}, + {"declare the index", "/v1/table/vec$vector$idx/declare", "", "read-only"}, + {"describe with an admin token", "/v1/table/vec$vector$idx/describe", forged, "read-only"}, + {"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, "read-write"}, + {"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, "read-write"}, + {"another namespace", "/v1/table/vec$other$tbl/describe", "", "read-write"}, + {"bucket from before formats", "/v1/table/old$vector$idx/describe", "", "read-write"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := h.vendedKey(t, tc.target, tc.token); got != tc.want { + t.Errorf("vended %q, want %q", got, tc.want) + } + }) + } +} + +// Declaring a name that already exists answers for that table, so naming the +// index's location on another table gets no credentials to the index. +func TestDeclareOfAnExistingTableKeepsItsLocation(t *testing.T) { + h := newTestHarness(t) + h.server.SetCredentialVendor(recordingVendor{}) + h.seedVectorIndex(t, "vec", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK) + + rec := h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", + `{"location":"s3://vec/vector/idx","vend_credentials":true}`, http.StatusOK) + resp := decode[DeclareTableResponse](t, rec) + if resp.Location != "s3://vec/other/tbl" { + t.Errorf("location = %q, want the table's own s3://vec/other/tbl", resp.Location) + } + if got := resp.StorageOptions["aws_secret_access_key"]; got != "other/tbl" { + t.Errorf("credentials cover %q, want other/tbl", got) + } + if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil { + t.Error("the declare wrote into the index") + } +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 8bd16cb1ec..281bbaa6ef 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1500,6 +1500,15 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{ return "", nil, err } +// FilerSigningKey is the current jwt.filer_signing.key, which follows a +// security.toml reload. +func (s3a *S3ApiServer) FilerSigningKey() security.SigningKey { + if s3a.filerGuard == nil { + return nil + } + return s3a.filerGuard.SigningKey() +} + // DefaultAllow returns whether access is allowed by default when no policy is found func (s3a *S3ApiServer) DefaultAllow() bool { if s3a.iam == nil || s3a.iam.iamIntegration == nil { diff --git a/weed/s3api/s3tables/handler_table.go b/weed/s3api/s3tables/handler_table.go index a9df3373d8..10edb2e94d 100644 --- a/weed/s3api/s3tables/handler_table.go +++ b/weed/s3api/s3tables/handler_table.go @@ -188,8 +188,9 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) resp := &CreateTableResponse{ - TableARN: tableARN, - VersionToken: versionToken, + TableARN: tableARN, + VersionToken: versionToken, + MetadataLocation: metadata.MetadataLocation, } h.writeJSON(w, http.StatusOK, resp) diff --git a/weed/security/grpc_peer.go b/weed/security/grpc_peer.go index a6b1fba95e..9a2844067d 100644 --- a/weed/security/grpc_peer.go +++ b/weed/security/grpc_peer.go @@ -38,16 +38,26 @@ func CheckGrpcAdminPeer(ctx context.Context, signingKey SigningKey) error { if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") || parts[1] == "" { return status.Error(codes.Unauthenticated, "authorization header must use Bearer scheme") } - parsed, err := DecodeJwt(signingKey, EncodedJwt(parts[1]), &SeaweedFilerAdminClaims{}) - if err != nil || parsed == nil || !parsed.Valid { - return status.Error(codes.Unauthenticated, "invalid admin token") - } - if claims, ok := parsed.Claims.(*SeaweedFilerAdminClaims); !ok || claims.SessionId != "" { + if !IsFilerAdminToken(signingKey, EncodedJwt(parts[1])) { return status.Error(codes.Unauthenticated, "invalid admin token") } return nil } +// IsFilerAdminToken reports whether token is an admin token signed by +// signingKey. An STS session token signed with the same key does not count. +func IsFilerAdminToken(signingKey SigningKey, token EncodedJwt) bool { + if len(signingKey) == 0 || token == "" { + return false + } + parsed, err := DecodeJwt(signingKey, token, &SeaweedFilerAdminClaims{}) + if err != nil || parsed == nil || !parsed.Valid { + return false + } + claims, ok := parsed.Claims.(*SeaweedFilerAdminClaims) + return ok && claims.SessionId == "" +} + // GrpcAdminPeerCredentials returns outgoing metadata carrying a Bearer token // for admin-plane gRPC handlers, or nil when no signing key is configured. func GrpcAdminPeerCredentials(signingKey SigningKey, expiresAfterSec int) metadata.MD { From 80e0bf97dd8dbdfb1f66aa32d87c44e62bbeb847 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:50:48 -0800 Subject: [PATCH 11/31] shell: dial volume servers on their registered gRPC port in volume.scrub, ec.scrub, volume.server.state (#12315) * shell: dial scrubbed volume servers on their registered gRPC port volume.scrub, ec.scrub and volume.server.state built each data node's address from its HTTP address alone, so a volume server whose gRPC port is not the HTTP port + 10000 was dialed on the wrong port. Co-Authored-By: Claude Opus 5.5 * shell: resolve -node volume servers to their registered gRPC port An explicit -node/-nodes list was dialed on HTTP port + 10000 too. Match each entry against the topology and take the data node's gRPC port; an address the master does not know is used as given. Co-Authored-By: Claude Opus 5.5 * shell: keep a gRPC port given in -node A node given as host:port.grpcPort was rewritten to the registered gRPC port; only a bare host:port takes it from the topology. Co-Authored-By: Claude Opus 5.5 * shell: dial -node volume servers when the topology is unavailable An explicit node list did not need the master before; keep using it as given if the volume list cannot be fetched. Selecting every node still requires the topology. Co-Authored-By: Claude Opus 5.5 * shell: bound the topology lookup for an explicit -node list With the master down, volumeServer.state -nodes waited on the master forever instead of reaching the given volume servers. Give the lookup a 30s deadline and fall back to the addresses as given. Co-Authored-By: Claude Opus 5.5 * shell: resolve volume server addresses from a topology-only listing Only the data nodes are needed to find each server's gRPC port, so ask the master for the topology without its volumes and ec shards. A large cluster no longer risks running past the -node lookup deadline and falling back to the HTTP port + 10000. Co-Authored-By: Claude Opus 5.5 * shell: skip the topology lookup when every -node names its gRPC port A host:port.grpcPort entry is dialed as given, so a list made only of those does not need the master and no longer waits on it during an outage. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/shell/command_ec_common.go | 85 ++++++++++++++++++++--- weed/shell/command_ec_scrub.go | 21 +----- weed/shell/command_volume_scrub.go | 15 +--- weed/shell/command_volume_scrub_test.go | 54 ++++++++++++++ weed/shell/command_volume_server_state.go | 16 +---- 5 files changed, 139 insertions(+), 52 deletions(-) create mode 100644 weed/shell/command_volume_scrub_test.go diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index ea7a9fd9e2..bf49b02b7c 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -93,10 +93,13 @@ func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Dura if delayBeforeCollecting > 0 { time.Sleep(delayBeforeCollecting) } + return fetchTopologyInfo(context.Background(), commandEnv, &master_pb.VolumeListRequest{}) +} +func fetchTopologyInfo(ctx context.Context, commandEnv *CommandEnv, request *master_pb.VolumeListRequest) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { var resp *master_pb.VolumeListResponse - err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error { - resp, err = pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{}) + err = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + resp, err = pb.CollectVolumeList(ctx, client, request) return err }) if err != nil { @@ -107,22 +110,88 @@ func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Dura } func collectDataNodes(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) ([]*master_pb.DataNodeInfo, error) { - dataNodes := []*master_pb.DataNodeInfo{} - topo, _, err := collectTopologyInfo(commandEnv, delayBeforeCollecting) if err != nil { return nil, err } + return topologyDataNodes(topo), nil +} +func topologyDataNodes(topo *master_pb.TopologyInfo) []*master_pb.DataNodeInfo { + dataNodes := []*master_pb.DataNodeInfo{} for _, dci := range topo.GetDataCenterInfos() { for _, r := range dci.GetRackInfos() { - for _, dn := range r.GetDataNodeInfos() { - dataNodes = append(dataNodes, dn) - } + dataNodes = append(dataNodes, r.GetDataNodeInfos()...) } } + return dataNodes +} - return dataNodes, nil +// collectVolumeServerAddresses resolves nodes with volumeServerAddresses from +// a topology-only listing. An explicit node list bounds the lookup and is used +// as given when the topology is unavailable, so it still works while the +// master is down. +func collectVolumeServerAddresses(commandEnv *CommandEnv, nodes string) ([]pb.ServerAddress, error) { + if nodes != "" && everyNodeNamesGrpcPort(nodes) { + return volumeServerAddresses(nil, nodes), nil + } + ctx := context.Background() + if nodes != "" { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + } + topo, _, err := fetchTopologyInfo(ctx, commandEnv, &master_pb.VolumeListRequest{TopologyOnly: true}) + if err != nil { + if nodes == "" { + return nil, err + } + glog.V(0).Infof("dialing volume servers as given, topology unavailable: %v", err) + } + return volumeServerAddresses(topologyDataNodes(topo), nodes), nil +} + +func everyNodeNamesGrpcPort(nodes string) bool { + for _, node := range strings.Split(nodes, ",") { + node = strings.TrimSpace(node) + if node == "" { + continue + } + portsSepIndex := strings.LastIndex(node, ":") + if portsSepIndex < 0 || !strings.Contains(node[portsSepIndex+1:], ".") { + return false + } + } + return true +} + +// volumeServerAddresses returns the comma-separated nodes, or every data node +// when nodes is empty, dialing each registered server on its gRPC port unless +// the node names one. +func volumeServerAddresses(dataNodes []*master_pb.DataNodeInfo, nodes string) []pb.ServerAddress { + registered := make(map[string]pb.ServerAddress, len(dataNodes)) + all := make([]pb.ServerAddress, 0, len(dataNodes)) + for _, dn := range dataNodes { + addr := pb.NewServerAddressFromDataNode(dn) + registered[addr.ToHttpAddress()] = addr + all = append(all, addr) + } + if nodes == "" { + return all + } + addrs := []pb.ServerAddress{} + for _, node := range strings.Split(nodes, ",") { + node = strings.TrimSpace(node) + if node == "" { + continue + } + if addr, found := registered[node]; found { + addrs = append(addrs, addr) + } else { + addrs = append(addrs, pb.ServerAddress(node)) + } + } + return addrs } func collectEcNodesForDC(commandEnv *CommandEnv, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { diff --git a/weed/shell/command_ec_scrub.go b/weed/shell/command_ec_scrub.go index 13414fba3e..00a1fa072d 100644 --- a/weed/shell/command_ec_scrub.go +++ b/weed/shell/command_ec_scrub.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/seaweedfs/seaweedfs/weed/ec" - "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" ) @@ -53,23 +52,9 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer return } - volumeServerAddrs := []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - addr = strings.TrimSpace(addr) - if addr == "" { - continue - } - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + volumeServerAddrs, err := collectVolumeServerAddresses(commandEnv, *nodesStr) + if err != nil { + return err } volumeIDs := []uint32{} diff --git a/weed/shell/command_volume_scrub.go b/weed/shell/command_volume_scrub.go index 95afd3a329..0671075eea 100644 --- a/weed/shell/command_volume_scrub.go +++ b/weed/shell/command_volume_scrub.go @@ -62,19 +62,8 @@ func (c *commandVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer io return } - c.volumeServerAddrs = []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + if c.volumeServerAddrs, err = collectVolumeServerAddresses(commandEnv, *nodesStr); err != nil { + return err } c.volumeIDs = []uint32{} diff --git a/weed/shell/command_volume_scrub_test.go b/weed/shell/command_volume_scrub_test.go new file mode 100644 index 0000000000..8a2d29d867 --- /dev/null +++ b/weed/shell/command_volume_scrub_test.go @@ -0,0 +1,54 @@ +package shell + +import ( + "reflect" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestVolumeServerAddressesUseRegisteredGrpcPort(t *testing.T) { + dataNodes := []*master_pb.DataNodeInfo{ + {Id: "10.0.0.1:8080", Address: "10.0.0.1:8080", GrpcPort: 18080}, + {Id: "10.0.0.2:7000", Address: "10.0.0.2:7000", GrpcPort: 7500}, + {Id: "10.0.0.3:8081"}, + } + tests := []struct { + nodes string + wantGrpc []string + }{ + {"", []string{"10.0.0.1:18080", "10.0.0.2:7500", "10.0.0.3:18081"}}, + {"10.0.0.2:7000", []string{"10.0.0.2:7500"}}, + {" 10.0.0.2:7000 ,,10.0.0.1:8080", []string{"10.0.0.2:7500", "10.0.0.1:18080"}}, + {"10.0.0.9:7000.7600", []string{"10.0.0.9:7600"}}, + {"10.0.0.2:7000.7600", []string{"10.0.0.2:7600"}}, + } + for _, tt := range tests { + var gotGrpc []string + for _, addr := range volumeServerAddresses(dataNodes, tt.nodes) { + gotGrpc = append(gotGrpc, addr.ToGrpcAddress()) + } + if !reflect.DeepEqual(gotGrpc, tt.wantGrpc) { + t.Errorf("volumeServerAddresses(%q) dials %v, want %v", tt.nodes, gotGrpc, tt.wantGrpc) + } + } +} + +func TestEveryNodeNamesGrpcPort(t *testing.T) { + tests := []struct { + nodes string + want bool + }{ + {"10.0.0.9:7000.7600", true}, + {" 10.0.0.9:7000.7600 ,,[::1]:8080.18080", true}, + {"10.0.0.9:7000.7600,10.0.0.2:7000", false}, + {"10.0.0.2:7000", false}, + {"volume.example.com", false}, + {"volume.example.com:8080", false}, + } + for _, tt := range tests { + if got := everyNodeNamesGrpcPort(tt.nodes); got != tt.want { + t.Errorf("everyNodeNamesGrpcPort(%q) = %v, want %v", tt.nodes, got, tt.want) + } + } +} diff --git a/weed/shell/command_volume_server_state.go b/weed/shell/command_volume_server_state.go index 9ab850f568..03fd5814f0 100644 --- a/weed/shell/command_volume_server_state.go +++ b/weed/shell/command_volume_server_state.go @@ -86,19 +86,9 @@ func (c *commandVolumeServerState) Do(args []string, commandEnv *CommandEnv, wri return fmt.Errorf("--maintenanceOn and --maintenanceOff are mutually exclusive") } - volumeServerAddrs := []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + volumeServerAddrs, err := collectVolumeServerAddresses(commandEnv, *nodesStr) + if err != nil { + return err } slices.Sort(volumeServerAddrs) if len(volumeServerAddrs) == 0 { From 54777d7fb91c0ad9d63ae3d7c521f193f0c8c638 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:56:09 -0800 Subject: [PATCH 12/31] vector: build weed-vector without a system protoc (#12317) * vector: build without a system protoc lance's build scripts run $PROTOC and fail without one. Point it at the protoc that protoc-bin-vendored ships, at the version in Cargo.lock, so building inside seaweed-vector/ needs only the Rust toolchain. An explicit PROTOC still wins, and a registry without the crate falls back to PATH. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): build weed-vector inside its workspace Cargo reads seaweed-vector/.cargo/config.toml only when it runs there, so build from that directory to pick up the vendored protoc. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * ci(vector): drop the PROTOC step the workspace config replaces Builds run inside seaweed-vector/, where the cargo config already points lance at the vendored protoc. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): resolve CARGO_TARGET_DIR before changing directory A relative CARGO_TARGET_DIR given to make would point cargo, which now runs in seaweed-vector/, and the cp after it at different directories. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * vector: pick the vendored protoc on every Linux host it ships for protoc-bin-vendored also carries ppc64le, s390x and x86_32 binaries. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/rust-vector-tests.yml | 22 ---------------------- seaweed-vector/.cargo/config.toml | 6 ++++++ seaweed-vector/.cargo/protoc | 21 +++++++++++++++++++++ test/s3vectors/Makefile | 3 ++- 4 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 seaweed-vector/.cargo/config.toml create mode 100755 seaweed-vector/.cargo/protoc diff --git a/.github/workflows/rust-vector-tests.yml b/.github/workflows/rust-vector-tests.yml index 1776ae513a..431b0d9d99 100644 --- a/.github/workflows/rust-vector-tests.yml +++ b/.github/workflows/rust-vector-tests.yml @@ -68,19 +68,6 @@ jobs: restore-keys: | rust-vector-${{ steps.toolchain.outputs.fingerprint }}- - # lance's build scripts compile their own protos and look for a protoc. - # Point them at the one protoc-bin-vendored ships, so no job depends on a - # system package and every build sees the same version. - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - - name: Build the vector crates run: cd seaweed-vector && cargo build --release @@ -125,15 +112,6 @@ jobs: key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }} restore-keys: | vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}- - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - name: Test Go vector gateway run: | go test ./weed/s3api/s3vectors -count=1 diff --git a/seaweed-vector/.cargo/config.toml b/seaweed-vector/.cargo/config.toml new file mode 100644 index 0000000000..d3b5a7e240 --- /dev/null +++ b/seaweed-vector/.cargo/config.toml @@ -0,0 +1,6 @@ +# lance's build scripts run $PROTOC. Point them at the protoc that +# protoc-bin-vendored ships, as this workspace's own build script does, so a +# build needs no system protoc. An explicit PROTOC still wins. Cargo reads this +# file only when it runs inside seaweed-vector/. +[env] +PROTOC = { value = ".cargo/protoc", relative = true } diff --git a/seaweed-vector/.cargo/protoc b/seaweed-vector/.cargo/protoc new file mode 100755 index 0000000000..3f1717c478 --- /dev/null +++ b/seaweed-vector/.cargo/protoc @@ -0,0 +1,21 @@ +#!/bin/sh +# Runs the protoc of the protoc-bin-vendored version in Cargo.lock, or the one +# on PATH when the registry does not hold it. +case "$(uname -s)-$(uname -m)" in +Linux-x86_64) platform=linux-x86_64 ;; +Linux-aarch64) platform=linux-aarch_64 ;; +Linux-ppc64le) platform=linux-ppcle_64 ;; +Linux-s390x) platform=linux-s390_64 ;; +Linux-i?86) platform=linux-x86_32 ;; +Darwin-x86_64) platform=macos-x86_64 ;; +Darwin-arm64) platform=macos-aarch_64 ;; +*) exec protoc "$@" ;; +esac +crate="protoc-bin-vendored-$platform" +version=$(awk -v name="\"$crate\"" '$1 == "name" && $3 == name { found = 1; next } found && $1 == "version" { gsub(/"/, "", $3); print $3; exit }' "$(dirname "$0")/../Cargo.lock") +for protoc in "${CARGO_HOME:-$HOME/.cargo}"/registry/src/*/"$crate-$version"/bin/protoc; do + if [ -x "$protoc" ]; then + exec "$protoc" "$@" + fi +done +exec protoc "$@" diff --git a/test/s3vectors/Makefile b/test/s3vectors/Makefile index b5d301243a..5a390bb950 100644 --- a/test/s3vectors/Makefile +++ b/test/s3vectors/Makefile @@ -7,6 +7,7 @@ endif VOLUME := ./weed-volume CARGO_PROFILE ?= release CARGO_TARGET_DIR ?= $(abspath ../../seaweed-vector/target) +override CARGO_TARGET_DIR := $(abspath $(CARGO_TARGET_DIR)) CARGO_OUTPUT := $(CARGO_TARGET_DIR)/$(if $(filter dev,$(CARGO_PROFILE)),debug,$(CARGO_PROFILE)) export CARGO_TARGET_DIR @@ -18,7 +19,7 @@ $(WEED): FORCE go build -o $(WEED) ../../weed $(VEC): FORCE - cargo build --locked --profile $(CARGO_PROFILE) --manifest-path ../../seaweed-vector/Cargo.toml -p weed-vector-server + cd ../../seaweed-vector && cargo build --locked --profile $(CARGO_PROFILE) -p weed-vector-server cp $(CARGO_OUTPUT)/weed-vector $(VEC) ifneq ($(WORKER),) From d6c0d728e45ff86e3c957311d1f11951e9b06bc6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 23:50:19 -0800 Subject: [PATCH 13/31] s3vectors: resolve the caller like table buckets and honor bucket-scoped identity actions (#12319) * s3vectors: resolve the caller the way table buckets do The vector checks used the identity name as principal, while the catalog records a bucket's owner through getAccountID: the account id, or the issuer-scoped user id of a federated identity. An identity with an account or a federated identity did not own the vector bucket it created, a Deny naming its account did not bind it, and an Admin identity in the admin account did not get the admin bypass it has on table buckets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3vectors: honor identity actions scoped to a vector bucket An identity action such as s3vectors:QueryVectors:team-a granted nothing, because the vector policy context never named the bucket. Table buckets match : through the same context field, so pass the bucket the check is about. Creating a bucket stays unscoped, as for table buckets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3tables/handler_principal_test.go | 25 +- weed/s3api/s3tables/manager.go | 23 +- weed/s3api/s3vectors/handlers.go | 17 +- weed/s3api/s3vectors/principal_test.go | 240 ++++++++++++++++++ 4 files changed, 280 insertions(+), 25 deletions(-) create mode 100644 weed/s3api/s3vectors/principal_test.go diff --git a/weed/s3api/s3tables/handler_principal_test.go b/weed/s3api/s3tables/handler_principal_test.go index 93014a78c3..eed0eb57ba 100644 --- a/weed/s3api/s3tables/handler_principal_test.go +++ b/weed/s3api/s3tables/handler_principal_test.go @@ -30,13 +30,28 @@ func TestPrincipalForMirrorsGetAccountID(t *testing.T) { m := NewManager() m.SetAccountID("000000000000") - assert.Equal(t, "000000000000", m.PrincipalFor("")) - assert.Equal(t, "alice", m.PrincipalFor("alice")) - assert.Equal(t, s3_constants.AccountAdminId, m.PrincipalFor(s3_constants.AccountAdminId), + named := func(name string) *http.Request { + req := httptest.NewRequest(http.MethodGet, "/", nil) + return req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), name)) + } + withIdentity := func(id *testIdentity) *http.Request { + req := named(id.Name) + return req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id)) + } + + assert.Equal(t, "000000000000", m.PrincipalFor(nil)) + assert.Equal(t, "000000000000", m.PrincipalFor(named(""))) + assert.Equal(t, "alice", m.PrincipalFor(named("alice"))) + assert.Equal(t, s3_constants.AccountAdminId, m.PrincipalFor(named(s3_constants.AccountAdminId)), "the name branch of getAccountID keeps the admin name") - assert.Equal(t, "000000000000", m.PrincipalFor(" ")) - assert.Equal(t, "bob", m.PrincipalFor("arn:aws:iam::123456789012:user/bob"), + assert.Equal(t, "000000000000", m.PrincipalFor(named(" "))) + assert.Equal(t, "bob", m.PrincipalFor(named("arn:aws:iam::123456789012:user/bob")), "ARN identities collapse to the trailing segment like getAccountID") + assert.Equal(t, "team-acct", m.PrincipalFor(withIdentity(&testIdentity{Name: "carol", Account: &testIdentityAccount{Id: "team-acct"}})), + "an identity with an account is its account, not its name") + assert.Equal(t, "uid-alice", m.PrincipalFor(withIdentity(&testIdentity{Name: "alice-sub", Account: &testIdentityAccount{Id: "alice-sub"}, + Claims: map[string]interface{}{"aws:userid": "uid-alice"}})), + "a federated identity is its issuer-scoped user id") } func TestCheckPermissionDenyPolicyBindsNonAdmin(t *testing.T) { diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index c051a5ecb9..0d6cb441e1 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -46,24 +46,25 @@ func (m *Manager) SetTrusted(trusted bool) { m.handler.SetTrusted(trusted) } -// PrincipalFor maps a forwarded identity name onto the account the catalog -// would charge the request to: a synthesized request resolves its identity -// name through getAccountID's name branch — normalized like any principal — -// and an empty name falls back to the handler's account ID. -func (m *Manager) PrincipalFor(identity string) string { - if principal := normalizePrincipalID(identity); principal != "" { - return principal +// PrincipalFor resolves the caller of r the way the catalog's handlers do, +// so it matches the owner CreateTableBucket records. A nil request is the +// handler's account ID. +func (m *Manager) PrincipalFor(r *http.Request) string { + if r == nil { + return m.handler.accountID } - return m.handler.accountID + return m.handler.getAccountID(r) } // PolicyContext builds the evaluation context a Manager call's synthesized // request produces: the configured local-account predicate, the caller's own // identity actions so admin and action grants resolve on privileges rather // than the forwarded name, plus the open-by-default fallback only for a -// trusted caller or no resolved identity. -func (m *Manager) PolicyContext(r *http.Request) *PolicyContext { - ctx := &PolicyContext{IsLocalAccountID: m.handler.localAccountIDs, IdentityActions: getIdentityActions(r)} +// trusted caller or no resolved identity. A non-empty bucket lets an identity +// action scoped to that bucket apply, as the handlers do. +func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext { + ctx := &PolicyContext{IsLocalAccountID: m.handler.localAccountIDs, IdentityActions: getIdentityActions(r), + TrustedIssuers: m.handler.trustedIssuers(r), TableBucketName: bucket} if m.handler.trusted || (m.handler.defaultAllow && s3_constants.GetIdentityNameFromContext(r) == "") { ctx.DefaultAllow = true } diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 77213ba2f9..2dbe75eab3 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -277,12 +277,11 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request } // No bucket exists yet, so check against the handler account as the // catalog does for CreateTableBucket. - principal := s.tables.PrincipalFor(identityOf(r)) - if !s3tables.CheckPermissionWithContext("s3vectors:CreateVectorBucket", principal, s.tables.PrincipalFor(""), "", "", s.tables.PolicyContext(r)) { + principal := s.tables.PrincipalFor(r) + if !s3tables.CheckPermissionWithContext("s3vectors:CreateVectorBucket", principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, "")) { catalogErr(w, errVectorDenied) return } - // The caller's name still makes it the owner of the record. var resp map[string]interface{} err := s.internal.Execute(r.Context(), s.filerClient, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ @@ -360,8 +359,7 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) catalogErr(w, err) return } - principal := s.tables.PrincipalFor(identityOf(r)) - pctx := s.tables.PolicyContext(r) + principal := s.tables.PrincipalFor(r) // The catalog token continues the filtered page: ICEBERG buckets drop out // of the listing, Lance and format-less legacy buckets stay. out := &ListVectorBucketsResponse{NextToken: resp.ContinuationToken, VectorBuckets: []*VectorBucket{}} @@ -376,6 +374,7 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) continue } arn := s.vectorBucketArn(b.Name) + pctx := s.tables.PolicyContext(r, b.Name) if !s3tables.CheckPermissionWithContext("s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) && !s3tables.CheckPermissionWithContext("s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) { continue @@ -753,9 +752,9 @@ var errVectorDenied = &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied // compare-and-swap, so a grant revoked between reads never authorizes the // pending write. func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket, owner string, ext map[string][]byte) bool { - principal := s.tables.PrincipalFor(identityOf(r)) + principal := s.tables.PrincipalFor(r) return s3tables.CheckPermissionWithContext(action, principal, owner, - vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r)) + vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r, bucket)) } // authorizeVector grants the request's own vector operation — not a catalog @@ -777,8 +776,8 @@ func (s *Server) authorizeVector(ctx context.Context, r *http.Request, action, b if index != "" { resourceARN = s.indexArn(bucket, index) } - principal := s.tables.PrincipalFor(identityOf(r)) - if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN, s.tables.PolicyContext(r)) { + principal := s.tables.PrincipalFor(r) + if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN, s.tables.PolicyContext(r, bucket)) { return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"} } if !isVectorBucket(record) { diff --git a/weed/s3api/s3vectors/principal_test.go b/weed/s3api/s3vectors/principal_test.go new file mode 100644 index 0000000000..7b3ee431a5 --- /dev/null +++ b/weed/s3api/s3vectors/principal_test.go @@ -0,0 +1,240 @@ +package s3vectors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +// authTestEnv drives the vector API through Auth with a chosen identity and +// the table bucket API through the catalog with the same identity, so the +// two can be compared decision by decision. +type authTestEnv struct { + t *testing.T + s *Server + auth *fakeAuthenticator + tables *s3tables.Manager + admin *s3tables.Manager + client s3tables.FilerClient +} + +func newAuthTestEnv(t *testing.T) *authTestEnv { + fs := s3tablestest.Start(t) + client := s3tables.NewManagerClient(fs.Client) + auth := &fakeAuthenticator{} + admin := s3tables.NewManager() + admin.SetTrusted(true) + return &authTestEnv{t: t, s: NewServer(client, auth, newFakeVectors(0)), auth: auth, + tables: s3tables.NewManager(), admin: admin, client: client} +} + +func (e *authTestEnv) vector(ident *s3api.Identity, handler http.HandlerFunc, payload interface{}) int { + e.t.Helper() + return e.vectorResp(ident, handler, payload).Code +} + +func (e *authTestEnv) vectorResp(ident *s3api.Identity, handler http.HandlerFunc, payload interface{}) *httptest.ResponseRecorder { + e.t.Helper() + body, err := json.Marshal(payload) + if err != nil { + e.t.Fatalf("marshal request: %v", err) + } + e.auth.identity = ident + w := httptest.NewRecorder() + e.s.Auth(handler)(w, httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))) + return w +} + +func (e *authTestEnv) table(ident *s3api.Identity, operation string, req interface{}) error { + ctx := s3_constants.SetIdentityInContext(contextWithIdentity(context.Background(), ident.Name), ident) + var resp map[string]interface{} + return e.tables.Execute(ctx, e.client, operation, req, &resp, ident.Name) +} + +func tableARN(bucket string) string { + return "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket +} + +func denyPolicy(service, principal string) string { + return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":%q},"Action":"%s:*","Resource":"*"}]}`, + principal, service) +} + +func (e *authTestEnv) createBuckets(ident *s3api.Identity, name string) { + e.t.Helper() + if err := e.table(ident, "CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "t-" + name}); err != nil { + e.t.Fatalf("%s: create table bucket: %v", ident.Name, err) + } + if code := e.vector(ident, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-" + name}); code != http.StatusOK { + e.t.Fatalf("%s: create vector bucket: %d", ident.Name, code) + } +} + +func (e *authTestEnv) deny(name, principal string) { + e.t.Helper() + var resp map[string]interface{} + if err := e.admin.Execute(context.Background(), e.client, "PutTableBucketPolicy", &s3tables.PutTableBucketPolicyRequest{ + TableBucketARN: tableARN("t-" + name), ResourcePolicy: denyPolicy("s3tables", principal), + }, &resp, ""); err != nil { + e.t.Fatalf("put table policy: %v", err) + } + if code := e.vector(adminIdentity(), e.s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{ + VectorBucketName: "v-" + name, Policy: denyPolicy("s3vectors", principal), + }); code != http.StatusOK { + e.t.Fatalf("put vector policy: %d", code) + } +} + +// canGet reports whether ident may read the table bucket and the vector +// bucket created under name. +func (e *authTestEnv) canGet(ident *s3api.Identity, name string) (table, vector bool) { + e.t.Helper() + table = e.table(ident, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: tableARN("t-" + name)}) == nil + vector = e.vector(ident, e.s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "v-" + name}) == http.StatusOK + return table, vector +} + +func adminIdentity() *s3api.Identity { + return &s3api.Identity{Name: "admin", Account: &s3api.AccountAdmin, Actions: []s3api.Action{s3_constants.ACTION_ADMIN}} +} + +var createActions = []s3api.Action{"s3tables:CreateTableBucket", "s3vectors:CreateVectorBucket"} + +// A vector bucket resolves its caller the way a table bucket does: an +// identity with an account owns what it created and is bound by a Deny +// naming its account, a federated identity is its issuer-scoped user id, and +// only the admin account's admin bypasses a Deny. +func TestVectorPrincipalMatchesTableBuckets(t *testing.T) { + cases := []struct { + name string + ident *s3api.Identity + principal string + owns bool + denied bool + }{ + {name: "admin", ident: adminIdentity(), principal: "admin", owns: true, denied: false}, + {name: "admin-action", ident: &s3api.Identity{Name: "ops", Account: &s3api.Account{Id: "ops"}, + Actions: []s3api.Action{s3_constants.ACTION_ADMIN}}, principal: "ops", owns: true, denied: true}, + {name: "account", ident: &s3api.Identity{Name: "carol", Account: &s3api.Account{Id: "team-acct"}, + Actions: createActions}, principal: "team-acct", owns: true, denied: true}, + {name: "federated", ident: &s3api.Identity{Name: "alice-sub", Account: &s3api.Account{Id: "alice-sub"}, + Claims: map[string]interface{}{"aws:userid": "uid-alice", "sub": "alice-sub"}, Actions: createActions}, + principal: "uid-alice", owns: true, denied: true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e := newAuthTestEnv(t) + e.createBuckets(c.ident, c.name) + table, vector := e.canGet(c.ident, c.name) + if table != c.owns || vector != c.owns { + t.Fatalf("creator access: table=%v vector=%v, want %v", table, vector, c.owns) + } + e.deny(c.name, c.principal) + table, vector = e.canGet(c.ident, c.name) + if table != !c.denied || vector != !c.denied { + t.Fatalf("under a deny on %q: table=%v vector=%v, want %v", c.principal, table, vector, !c.denied) + } + }) + } +} + +func (e *authTestEnv) createIndex(bucket string) { + e.t.Helper() + if code := e.vector(adminIdentity(), e.s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: bucket, IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }); code != http.StatusOK { + e.t.Fatalf("create index in %s: %d", bucket, code) + } +} + +func (e *authTestEnv) listedVectorBuckets(ident *s3api.Identity) []string { + e.t.Helper() + w := e.vectorResp(ident, e.s.handleListVectorBuckets, &ListVectorBucketsRequest{}) + if w.Code != http.StatusOK { + e.t.Fatalf("list vector buckets: %d %s", w.Code, w.Body.String()) + } + var resp ListVectorBucketsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + e.t.Fatalf("decode list: %v", err) + } + names := []string{} + for _, b := range resp.VectorBuckets { + names = append(names, b.VectorBucketName) + } + return names +} + +// An identity action scoped to a bucket, s3vectors::, grants that +// operation on that vector bucket and its indexes only, as +// s3tables:: does on a table bucket. A Deny still wins, and the +// scoped form never creates a bucket. +func TestBucketScopedIdentityActions(t *testing.T) { + e := newAuthTestEnv(t) + for _, name := range []string{"team-a", "team-b"} { + e.createBuckets(adminIdentity(), name) + e.createIndex("v-" + name) + } + scoped := &s3api.Identity{Name: "scoped", Account: &s3api.Account{Id: "scoped"}, Actions: []s3api.Action{ + "s3tables:GetTableBucket:t-team-a", "s3tables:CreateTableBucket:t-team-c", + "s3vectors:GetVectorBucket:v-team-a", "s3vectors:GetIndex:v-team-a", "s3vectors:CreateVectorBucket:v-team-c", + }} + + if table, vector := e.canGet(scoped, "team-a"); !table || !vector { + t.Fatalf("granted bucket: table=%v vector=%v, want both", table, vector) + } + if table, vector := e.canGet(scoped, "team-b"); table || vector { + t.Fatalf("other bucket: table=%v vector=%v, want neither", table, vector) + } + getIndex := func(ident *s3api.Identity, bucket string) int { + return e.vector(ident, e.s.handleGetIndex, &GetIndexRequest{VectorBucketName: bucket, IndexName: "idx"}) + } + getVectors := func(ident *s3api.Identity, bucket string) int { + return e.vector(ident, e.s.handleGetVectors, &GetVectorsRequest{VectorBucketName: bucket, IndexName: "idx", Keys: []string{"a"}}) + } + if code := getIndex(scoped, "v-team-a"); code != http.StatusOK { + t.Fatalf("GetIndex in granted bucket: %d", code) + } + if code := getIndex(scoped, "v-team-b"); code != http.StatusForbidden { + t.Fatalf("GetIndex in other bucket: %d, want 403", code) + } + if code := getVectors(scoped, "v-team-a"); code != http.StatusForbidden { + t.Fatalf("ungranted GetVectors in granted bucket: %d, want 403", code) + } + if got := e.listedVectorBuckets(scoped); len(got) != 1 || got[0] != "v-team-a" { + t.Fatalf("listing: %v, want [v-team-a]", got) + } + if err := e.table(scoped, "CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "t-team-c"}); err == nil { + t.Fatalf("scoped CreateTableBucket created a table bucket") + } + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-team-c"}); code != http.StatusForbidden { + t.Fatalf("scoped CreateVectorBucket: %d, want 403", code) + } + + wild := &s3api.Identity{Name: "wild", Account: &s3api.Account{Id: "wild"}, Actions: []s3api.Action{"s3vectors:*:v-team-a"}} + if code := getVectors(wild, "v-team-a"); code != http.StatusOK { + t.Fatalf("wildcard grant in its bucket: %d", code) + } + if code := getIndex(wild, "v-team-b"); code != http.StatusForbidden { + t.Fatalf("wildcard grant in other bucket: %d, want 403", code) + } + if code := e.vector(wild, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-team-c"}); code != http.StatusForbidden { + t.Fatalf("wildcard bucket grant created a bucket: %d, want 403", code) + } + + e.deny("team-a", "scoped") + if table, vector := e.canGet(scoped, "team-a"); table || vector { + t.Fatalf("granted bucket under a deny: table=%v vector=%v, want neither", table, vector) + } + if code := getIndex(scoped, "v-team-a"); code != http.StatusForbidden { + t.Fatalf("GetIndex under a deny: %d, want 403", code) + } +} From ff7045a4e52264e9119f8b4cd3ec1709716d0b0e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 23:50:46 -0800 Subject: [PATCH 14/31] s3vectors: report the Euclidean distance, not its square (#12318) A euclidean index answered QueryVectors with Lance's L2 distance, which is the squared Euclidean distance: [3, 4] was 25 from the origin, not 5. AWS defines the metric as the straight-line distance between the vectors. The gateway now returns its square root, so the single engine and the enterprise coordinator, which both hand back the raw Lance value, report the same distance. Ordering is unchanged. Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK Co-authored-by: Claude Opus 5.5 --- test/s3vectors/s3vectors_test.py | 9 +++- weed/s3api/s3vectors/distance_metric_test.go | 52 ++++++++++++++++++++ weed/s3api/s3vectors/handlers.go | 8 ++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index e8138e7e35..53ce40c4dd 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -58,7 +58,7 @@ def test_vector_lifecycle(vectors): assert [h["key"] for h in hits] == ["a", "c"] distances = {h["key"]: h["distance"] for h in hits} assert distances["a"] == pytest.approx(0.0, abs=1e-6) - assert distances["c"] == pytest.approx(0.02, abs=1e-4) + assert distances["c"] == pytest.approx(0.02 ** 0.5, abs=1e-4) assert all(h["metadata"]["color"] == "red" for h in hits) vectors.delete_vectors(vectorBucketName=BUCKET, indexName=INDEX, keys=["b"]) @@ -223,6 +223,13 @@ def test_filter_arrays(vectors, index): assert matches({"$nin": [1, "2.5"]}) == ["flag", "scalar", "space"] +def test_euclidean_distance(vectors, index): + vectors.put_vectors(**index, vectors=[{"key": "p", "data": {"float32": [3, 4, 0, 0]}}]) + result = vectors.query_vectors(**index, queryVector={"float32": [0, 0, 0, 0]}, topK=1, returnDistance=True) + assert result["distanceMetric"] == "euclidean" + assert result["vectors"][0]["distance"] == pytest.approx(5.0) + + def test_cosine_metric(vectors): bucket = "cosine-" + uuid.uuid4().hex[:16] vectors.create_vector_bucket(vectorBucketName=bucket) diff --git a/weed/s3api/s3vectors/distance_metric_test.go b/weed/s3api/s3vectors/distance_metric_test.go index e3a149778e..502216bb8c 100644 --- a/weed/s3api/s3vectors/distance_metric_test.go +++ b/weed/s3api/s3vectors/distance_metric_test.go @@ -3,6 +3,7 @@ package s3vectors import ( "context" "encoding/json" + "fmt" "net/http" "testing" @@ -86,3 +87,54 @@ func TestIndexFromRecordLowercasesMetric(t *testing.T) { } } } + +type distancesFake struct { + *fakeVectors + distances []float32 +} + +func (f distancesFake) QueryVectors(ctx context.Context, req *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (*vector_pb.QueryVectorsResponse, error) { + resp := &vector_pb.QueryVectorsResponse{} + for i, d := range f.distances { + resp.Vectors = append(resp.Vectors, &vector_pb.QueryResult{Key: fmt.Sprint(i), Distance: d}) + } + return resp, nil +} + +// The engine returns Lance's squared L2 distance; a euclidean index reports +// its square root, the other metrics their engine value. +func TestQueryVectorsReportsEuclideanDistance(t *testing.T) { + for metric, want := range map[string][]float32{ + "euclidean": {5, 0.5, 0}, + "cosine": {25, 0.25, -1e-7}, + "dotproduct": {25, 0.25, -1e-7}, + } { + fs := s3tablestest.Start(t) + engine := distancesFake{newFakeVectors(0), []float32{25, 0.25, -1e-7}} + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 2, DistanceMetric: metric, + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + w := postJSON(t, s.handleQueryVectors, &QueryVectorsRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", + QueryVector: &QueryVector{Float32: []float32{1, 0}}, TopK: 3, ReturnDistance: true, + }) + var query QueryVectorsResponse + if err := json.Unmarshal(w.Body.Bytes(), &query); err != nil || len(query.Vectors) != len(want) { + t.Fatalf("%s QueryVectors: %d %s", metric, w.Code, w.Body.String()) + } + for i, v := range query.Vectors { + if v.Distance == nil { + t.Fatalf("%s hit %d has no distance", metric, i) + } + if *v.Distance != want[i] { + t.Fatalf("%s hit %d distance %v, want %v", metric, i, *v.Distance, want[i]) + } + } + } +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 2dbe75eab3..593e4237ce 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "net/http" "strconv" "strings" @@ -1938,7 +1939,12 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { for _, v := range resp.Vectors { o := &QueryOutputVector{Key: v.Key} if req.ReturnDistance { - o.Distance = &v.Distance + d := v.Distance + if metric == "euclidean" { + // Lance's L2 distance is the square of the Euclidean distance. + d = float32(math.Sqrt(math.Max(0, float64(d)))) + } + o.Distance = &d } if req.ReturnMetadata { o.Metadata = structToMap(v.Metadata) From d72f761d83e815d59e912869dd389e8dd90f5e38 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 23:52:30 -0800 Subject: [PATCH 15/31] seaweed-vector: work on its defaults, and without identities or credentials (#12323) * seaweed-vector: default --namespace to the gateway's Lance port The old default, http://localhost:8888/lance, is a path nothing serves: the Lance Namespace listens on its own port, 9101 by default on weed s3, weed server, weed filer and weed mini. With it, the filer answered the engine's catalog calls with a 500 and every CreateIndex failed with InternalFailure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: default --listen to a port no SeaweedFS server takes 127.0.0.1:8080 is the volume server's default port, so weed-vector next to weed server exited with "Address already in use (os error 98)". 8090 is not among the defaults of master, volume, filer, S3 and its Iceberg and Lance catalogs, IAM, WebDAV, SFTP, the message queue broker and agent, the admin server or weed mini, nor any of their gRPC ports (HTTP + 10000). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * lance: take any key while the S3 server runs without identities Without identities the S3 port accepts any signature and treats the caller as the admin. The Lance catalog still checked an x-api-key or Bearer token against identities that did not exist, so weed-vector started with --storage-option credentials on such a server got 401 on every catalog call. The catalog now skips both checks when S3 auth is off and leaves the caller to the S3 authenticator, which admits it as the admin, as it already did for a request with no key. With identities configured nothing changes: an unknown key is still 401. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * lance: tell clients to skip signing while S3 auth is off A client with no credentials of its own could not reach a table's data on a server without identities, though the S3 port takes unsigned requests there: lance fell back to the AWS default credential chain and failed with CredentialsNotLoaded. weed-vector without --storage-option failed every CreateIndex that way. The catalog now adds aws_skip_signature to the storage options it hands out while S3 auth is off, and leaves it out once identities exist. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: bind the listener before registering with the master The engine spawned its master registration and then bound --listen, so an engine whose port was taken could reach the master before it exited: on weed server with 18080 taken the master logged "vector node 127.0.0.1:18080 added" and removed it in the same millisecond, and the gateway noticed it. Bind first, and name the address in the error instead of "transport error". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-worker: keep an unsigned store off the AWS default chain A store the catalog says takes unsigned requests needs no credentials, but lance builds the AWS default credential chain for any store without them, and the chain probes instance metadata on every dataset open. With no credentials of its own, weed-vector spent a second on each query (1030-1066 ms against 32-46 ms with --storage-option ones). The merge of the operator's fallback and the vended options, which the engine and the Lance worker each did in their own copy, moves into table_options, and gives such a store empty credentials. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): serve an index with no identities and no engine credentials A second cluster runs weed s3 without -config and weed-vector without --storage-option, with no AWS credentials in either environment: weed s3 would turn them into an admin identity, and the engine would find them. On master its CreateIndex fails with InternalFailure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- seaweed-vector/crates/server/src/main.rs | 16 +++-- seaweed-vector/crates/server/src/service.rs | 3 +- seaweed-worker/crates/lance/src/dataset.rs | 70 +++++++++++++++++---- test/s3vectors/conftest.py | 34 +++++++--- test/s3vectors/s3vectors_test.py | 15 +++++ weed/s3api/lance/handlers_oauth_test.go | 33 +++++++++- weed/s3api/lance/handlers_test.go | 32 ++++++++++ weed/s3api/lance/server.go | 13 +++- weed/s3api/lance/storage.go | 5 ++ weed/s3api/s3api_server.go | 6 ++ weed/s3api/s3api_tables_auth_test.go | 28 +++++++++ 11 files changed, 225 insertions(+), 30 deletions(-) diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index aa42defcc7..42bbc9ca3b 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -1,6 +1,7 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use clap::Parser; use tonic::transport::Server; +use tonic::transport::server::TcpIncoming; use weed_lance_worker::catalog::{NamespaceClient, SigningKey}; use weed_lance_worker::dataset::FallbackOptions; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; @@ -10,15 +11,15 @@ use weed_vector_server::service::VectorServer; /// The VectorService the S3 gateway fans vector calls out to. #[derive(Parser)] struct Args { - /// Lance Namespace base URL, e.g. http://localhost:8888/lance. + /// Lance Namespace base URL: the S3 gateway's Lance port (-s3.port.lance). #[arg( long, env = "WEED_VECTOR_NAMESPACE", - default_value = "http://localhost:8888/lance" + default_value = "http://localhost:9101" )] namespace: String, /// gRPC listen address. - #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8080")] + #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")] listen: String, /// object_store option fallback for deployments that vend no credentials: /// key=value, repeatable. @@ -104,6 +105,11 @@ fn advertised_addr(listen: &str, masters: &[String]) -> String { async fn main() -> Result<()> { tracing_subscriber::fmt::init(); let args = Args::parse_from(normalize_args(std::env::args().collect())); + // Bind before registering, so the master never lists an engine that is + // about to exit because its port is taken. + let incoming = TcpIncoming::bind(args.listen.parse()?) + .with_context(|| format!("listen on {}", args.listen))? + .with_nodelay(Some(true)); if let Some(masters) = &args.masters { let masters: Vec = masters.split(',').map(|s| s.trim().to_string()).collect(); for master in &masters { @@ -132,7 +138,7 @@ async fn main() -> Result<()> { .max_decoding_message_size(1 << 30) .max_encoding_message_size(1 << 30), ) - .serve(args.listen.parse()?) + .serve_with_incoming(incoming) .await?; Ok(()) } diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index ea26f6c9bf..8ccf041c2b 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -430,8 +430,7 @@ impl VectorService for VectorServer { .describe_table(&table.id) .await .map_err(internal)?; - let mut options = self.fallback.clone(); - options.extend(description.storage_options.clone()); + let options = dataset::table_options(&description, &self.fallback); let schema = schema_for(dimension, metric, &req.non_filterable_metadata_keys); let empty = RecordBatch::new_empty(schema.clone()); let reader = arrow_array::RecordBatchIterator::new(vec![Ok(empty)].into_iter(), schema); diff --git a/seaweed-worker/crates/lance/src/dataset.rs b/seaweed-worker/crates/lance/src/dataset.rs index d28bcf2d4c..cdca46d0bd 100644 --- a/seaweed-worker/crates/lance/src/dataset.rs +++ b/seaweed-worker/crates/lance/src/dataset.rs @@ -65,21 +65,38 @@ pub async fn open_with( }) } -/// The namespace vends object_store's own option names, so they pass straight -/// through. What it vends always wins: the fallback exists because a deployment -/// without STS vends no credentials at all, and then the worker has no other way -/// to reach the data. +/// The object_store options for a table. The namespace vends object_store's own +/// option names, so they pass straight through. What it vends always wins: the +/// fallback exists because a deployment without STS vends no credentials at +/// all, and then the worker has no other way to reach the data. +pub fn table_options( + description: &TableDescription, + fallback: &FallbackOptions, +) -> HashMap { + let mut options: HashMap = fallback.clone(); + options.extend(description.storage_options.clone()); + // A store that takes unsigned requests needs no credentials, but without + // any lance builds the AWS default chain, and that probes instance + // metadata on every open. + if options + .get("aws_skip_signature") + .is_some_and(|v| v == "true") + { + options.entry("aws_access_key_id".into()).or_default(); + options.entry("aws_secret_access_key".into()).or_default(); + } + options +} + async fn open_at( description: &TableDescription, fallback: &FallbackOptions, wrapper: Option>, ) -> Result { - let mut options: HashMap = fallback.clone(); - options.extend(description.storage_options.clone()); DatasetBuilder::from_uri(&description.location) .with_store_params(ObjectStoreParams { storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options, + table_options(description, fallback), ))), object_store_wrapper: wrapper, ..Default::default() @@ -102,14 +119,12 @@ pub async fn drop( Err(err) => return Err(err), }; check_drop_location(id, &description.location)?; - let mut options: HashMap = fallback.clone(); - options.extend(description.storage_options.clone()); let (store, path) = ObjectStore::from_uri_and_params( Arc::new(ObjectStoreRegistry::default()), &description.location, &ObjectStoreParams { storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options, + table_options(&description, fallback), ))), ..Default::default() }, @@ -203,12 +218,45 @@ fn schema_json(dataset: &Dataset) -> Option { #[cfg(test)] mod tests { - use super::{check_drop_location, table_location}; + use super::{FallbackOptions, check_drop_location, table_location, table_options}; + use crate::catalog::TableDescription; fn id(parts: &[&str]) -> Vec { parts.iter().map(|part| part.to_string()).collect() } + fn options(pairs: &[(&str, &str)]) -> FallbackOptions { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn table_options_keep_an_unsigned_store_off_the_aws_default_chain() { + let described = |vended: &[(&str, &str)]| TableDescription { + location: "s3://bkt/vector/x".into(), + storage_options: options(vended), + managed_versioning: false, + }; + let unsigned = described(&[("aws_skip_signature", "true")]); + + let got = table_options(&unsigned, &FallbackOptions::new()); + assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("")); + assert_eq!( + got.get("aws_secret_access_key").map(String::as_str), + Some("") + ); + + let fallback = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]); + let got = table_options(&unsigned, &fallback); + assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("k")); + + let signed = described(&[("aws_endpoint", "http://s3:8333")]); + let got = table_options(&signed, &FallbackOptions::new()); + assert!(!got.contains_key("aws_access_key_id"), "{got:?}"); + } + #[test] fn table_location_matches_the_gateway_layout() { assert_eq!( diff --git a/test/s3vectors/conftest.py b/test/s3vectors/conftest.py index dfb5ab116d..69f4831bba 100644 --- a/test/s3vectors/conftest.py +++ b/test/s3vectors/conftest.py @@ -70,8 +70,18 @@ def stop(process): @pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p)) def cluster(request, tmp_path_factory): - volume, mode = request.param - directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}") + yield from run_cluster(request.param, tmp_path_factory, identities=True) + + +@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p)) +def open_cluster(request, tmp_path_factory): + """A cluster with no S3 identities, and an engine given no credentials.""" + yield from run_cluster(request.param, tmp_path_factory, identities=False) + + +def run_cluster(param, tmp_path_factory, identities): + volume, mode = param + directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}" + ("" if identities else "-open")) print(f"\nVector integration logs: {directory}") weed = binary("WEED_BINARY", "weed") engine = binary("VECTOR_BINARY", "weed-vector") @@ -89,10 +99,12 @@ def cluster(request, tmp_path_factory): master_addr = f"127.0.0.1:{master}.{master_grpc}" filer_addr = f"127.0.0.1:{filer}.{filer_grpc}" - def start(name, args): + def start(name, args, unset=()): log = stack.enter_context((directory / f"{name}.log").open("w")) env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test", AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true") + for key in unset: + env.pop(key) process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env) processes[name] = process stack.callback(stop, process) @@ -117,14 +129,20 @@ def cluster(request, tmp_path_factory): f"-defaultStoreDir={directory / 'filer'}"]) wait_for(lambda: listening(filer_grpc), processes, "filer not ready") endpoint = f"http://127.0.0.1:{s3}" - start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", f"-config={HERE / 's3.json'}", + # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity, and the engine + # would find it as a credential. + if identities: + config, unset = [f"-config={HERE / 's3.json'}"], () + credentials = ["--storage-option", "aws_access_key_id=test", + "--storage-option", "aws_secret_access_key=test"] + else: + config, unset, credentials = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), [] + start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", *config, "-ip=127.0.0.1", "-ip.bind=127.0.0.1", f"-port={s3}", - f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"]) + f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"], unset) wait_for(lambda: http_ready(f"http://127.0.0.1:{lance}/v1/namespace/root/list"), processes, "catalog not ready") start("vector", [engine, "--namespace", f"http://127.0.0.1:{lance}", - "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", - "--storage-option", "aws_access_key_id=test", - "--storage-option", "aws_secret_access_key=test"]) + "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *credentials], unset) wait_for(lambda: listening(vector), processes, "vector engine not ready") wait_for( diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index 53ce40c4dd..adbeb1c246 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -5,6 +5,7 @@ import time import uuid import boto3 +from botocore.config import Config from botocore.exceptions import ClientError import pytest @@ -415,3 +416,17 @@ def test_query_without_writer(vectors, index, cluster): raise time.sleep(0.2) assert [v["key"] for v in result["vectors"]] == ["survivor"] + + +def test_engine_without_credentials(open_cluster): + """With no S3 identities, an engine given no credentials still serves an index.""" + client = boto3.client("s3vectors", endpoint_url=open_cluster["endpoint"], aws_access_key_id="any", + aws_secret_access_key="any", region_name="us-east-1", + config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0})) + index = {"vectorBucketName": BUCKET, "indexName": INDEX} + client.create_vector_bucket(vectorBucketName=BUCKET) + client.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine") + client.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [1, 0]}}, + {"key": "b", "data": {"float32": [0, 1]}}]) + hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"] + assert [h["key"] for h in hits] == ["a"] diff --git a/weed/s3api/lance/handlers_oauth_test.go b/weed/s3api/lance/handlers_oauth_test.go index 236b101f32..c16e47ce0e 100644 --- a/weed/s3api/lance/handlers_oauth_test.go +++ b/weed/s3api/lance/handlers_oauth_test.go @@ -49,8 +49,9 @@ func newTestServerWithOAuth() *Server { } type mockS3Authenticator struct { - called bool - errCode s3err.ErrorCode + called bool + errCode s3err.ErrorCode + authDisabled bool } func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { @@ -63,6 +64,8 @@ func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, inte func (m *mockS3Authenticator) DefaultAllow() bool { return false } +func (m *mockS3Authenticator) AuthEnabled() bool { return !m.authDisabled } + func mintTestToken(t *testing.T, accessKey, secret string, issuedAt, expiresAt time.Time) string { t.Helper() key := deriveSigningKey(accessKey, secret) @@ -316,6 +319,32 @@ func TestAuthApiKeyInvalid(t *testing.T) { } } +// Without identities the S3 port takes any signature, so the catalog takes any +// key or token and leaves the caller to the S3 authenticator. +func TestAuthTakesAnyKeyWhenS3AuthIsOff(t *testing.T) { + s := newTestServerWithOAuth() + s.authenticator = &mockS3Authenticator{authDisabled: true} + + for header, value := range map[string]string{ + "x-api-key": "unknown:key", + "Authorization": "Bearer not-a-token", + } { + var gotIdentity string + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotIdentity = s3_constants.GetIdentityNameFromContext(r) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + req.Header.Set(header, value) + rec := httptest.NewRecorder() + handler(rec, req) + + if rec.Code != http.StatusOK || gotIdentity != "s3user" { + t.Fatalf("%s: status = %d, identity = %q, want 200 as s3user", header, rec.Code, gotIdentity) + } + } +} + func TestAuthNoBearerStillUsesS3Authenticator(t *testing.T) { s := newTestServerWithOAuth() auth := &mockS3Authenticator{errCode: s3err.ErrNone} diff --git a/weed/s3api/lance/handlers_test.go b/weed/s3api/lance/handlers_test.go index 72d9b84f68..037f66ff83 100644 --- a/weed/s3api/lance/handlers_test.go +++ b/weed/s3api/lance/handlers_test.go @@ -24,6 +24,14 @@ func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{} func (openAuthenticator) DefaultAllow() bool { return true } +func (openAuthenticator) AuthEnabled() bool { return false } + +// signedAuthenticator admits the same caller on a gateway that checks +// credentials. +type signedAuthenticator struct{ openAuthenticator } + +func (signedAuthenticator) AuthEnabled() bool { return true } + type testHarness struct { router *mux.Router filer *s3tablestest.MemFiler @@ -134,6 +142,28 @@ func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T { return out } +// While the S3 port takes unsigned requests the catalog says so, and a client +// with no credentials of its own still reaches the data. Once auth is on it +// has to sign again. +func TestStorageOptionsSkipSignatureOnlyWhileS3AuthIsOff(t *testing.T) { + h := newTestHarness(t) + h.createBucket(t, "open", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/namespace/open$ns/create", `{}`, http.StatusOK) + + declared := decode[DeclareTableResponse](t, + h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/declare", `{}`, http.StatusOK)) + if declared.StorageOptions["aws_skip_signature"] != "true" { + t.Fatalf("auth off: storage options = %v, want aws_skip_signature", declared.StorageOptions) + } + + h.server.authenticator = signedAuthenticator{} + described := decode[DescribeTableResponse](t, + h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/describe", `{}`, http.StatusOK)) + if _, ok := described.StorageOptions["aws_skip_signature"]; ok { + t.Fatalf("auth on: storage options = %v, want no aws_skip_signature", described.StorageOptions) + } +} + // The lifecycle a Lance client drives: create the namespace, declare the table // before any data exists, resolve it to a location, then deregister and bring it // back by registering the same location. @@ -429,6 +459,8 @@ func (createOnlyAuthenticator) AuthenticateRequest(*http.Request) (string, inter func (createOnlyAuthenticator) DefaultAllow() bool { return false } +func (createOnlyAuthenticator) AuthEnabled() bool { return true } + // The right to create a name is the write evidence for a table the declare // creates, but not for one it only finds. func TestDeclareVendsWriteOnlyForATableItCreates(t *testing.T) { diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index 5d25e8c860..286205a70c 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -25,6 +25,8 @@ type FilerClient interface { type S3Authenticator interface { AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) DefaultAllow() bool + // AuthEnabled reports whether the S3 server checks credentials at all. + AuthEnabled() bool } // CredentialValidator validates S3 access key / secret key pairs @@ -177,6 +179,13 @@ func loggingMiddleware(next http.Handler) http.Handler { }) } +// authEnabled is false when the S3 server has no identities and takes any +// signature. The catalog then takes any key too, and leaves the caller to the +// S3 authenticator. +func (s *Server) authEnabled() bool { + return s.authenticator == nil || s.authenticator.AuthEnabled() +} + // Auth authenticates the caller and puts the identity in the request context. // Lance clients authenticate the catalog with a Bearer token or an x-api-key // header; the S3 authenticator stays for callers that can SigV4-sign. @@ -190,7 +199,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // authenticator would parse the header as SigV4 and fail with a // different error that clients cannot act on. // The auth scheme is case-insensitive (RFC 7235). - if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { + if s.authEnabled() && strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) @@ -208,7 +217,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return } - if apiKey := r.Header.Get("x-api-key"); apiKey != "" { + if apiKey := r.Header.Get("x-api-key"); apiKey != "" && s.authEnabled() { if identityName, identity, ok := s.authenticateApiKey(apiKey); ok { accessKey, secretKey, _ := strings.Cut(apiKey, ":") ctx := r.Context() diff --git a/weed/s3api/lance/storage.go b/weed/s3api/lance/storage.go index 52a35640b5..e719680118 100644 --- a/weed/s3api/lance/storage.go +++ b/weed/s3api/lance/storage.go @@ -149,6 +149,11 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b if s.s3Region != "" { options["aws_region"] = s.s3Region } + if !s.authEnabled() { + // The S3 port takes unsigned requests, and a client with no + // credentials of its own would otherwise fail looking for some. + options["aws_skip_signature"] = "true" + } if !vend || s.credentialVendor == nil { return options, nil diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 281bbaa6ef..b6f8c1486a 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1509,6 +1509,12 @@ func (s3a *S3ApiServer) FilerSigningKey() security.SigningKey { return s3a.filerGuard.SigningKey() } +// AuthEnabled reports whether requests must carry valid credentials. With no +// identities configured, every request is allowed as the admin. +func (s3a *S3ApiServer) AuthEnabled() bool { + return s3a.iam == nil || s3a.iam.isEnabled() +} + // DefaultAllow returns whether access is allowed by default when no policy is found func (s3a *S3ApiServer) DefaultAllow() bool { if s3a.iam == nil || s3a.iam.iamIntegration == nil { diff --git a/weed/s3api/s3api_tables_auth_test.go b/weed/s3api/s3api_tables_auth_test.go index 6ab18d59d1..85080f5a69 100644 --- a/weed/s3api/s3api_tables_auth_test.go +++ b/weed/s3api/s3api_tables_auth_test.go @@ -159,6 +159,34 @@ func TestLanceAuthRejectsFailedSignature(t *testing.T) { assert.NotEqual(t, http.StatusOK, rr.Code) } +// The Lance catalog takes any x-api-key while the S3 port runs without +// identities, as the admin, and refuses an unknown key once auth is on. +func TestLanceAuthTakesAnyKeyOnlyWhileS3AuthIsOff(t *testing.T) { + s3a := &S3ApiServer{iam: &IdentityAccessManagement{}} + server := lance.NewServer(nil, s3a) + server.SetCredentialValidator(s3a) + + call := func() (int, string) { + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/list", nil) + req.Header.Set("x-api-key", "unknown:key") + identity := "" + rr := httptest.NewRecorder() + server.Auth(func(w http.ResponseWriter, r *http.Request) { + identity = s3_constants.GetIdentityNameFromContext(r) + w.WriteHeader(http.StatusOK) + })(rr, req) + return rr.Code, identity + } + + code, identity := call() + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, s3_constants.AccountAdminId, identity) + + s3a.iam.isAuthEnabled = true + code, _ = call() + assert.Equal(t, http.StatusUnauthorized, code) +} + func TestAuthSignatureOnlyScrubsAccountHeader(t *testing.T) { s3a := setupDefaultAllowAuthServer(t) From 2eb219df612e21dc347054eb2e7cac495034a3ad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 23:57:40 -0800 Subject: [PATCH 16/31] s3vectors: encrypt the objects of a vector bucket declared AES256 (#12324) * s3api: encrypt the objects of a vector bucket declared AES256 CreateVectorBucket recorded encryptionConfiguration sseType AES256 and returned it, but nothing on the write path read it: the index's data files, manifests and transactions were stored without SSE. A LANCE table bucket's record now supplies the bucket's default encryption whenever the bucket has none of its own, so PutObject, CreateMultipartUpload and CopyObject apply SSE-S3 to every object the engine writes. The S3 bucket metadata itself is unchanged. ICEBERG table buckets keep recording the declaration only: their catalog reads the manifests clients upload straight from the filer, which would see ciphertext. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3tables: refuse an AES256 vector bucket on a gateway without an SSE-S3 key A vector bucket's AES256 is now applied to every index write, and SSE-S3 fails each write when the gateway has no KEK. Creating the bucket there succeeded and the first CreateIndex came back as InternalFailure. CreateTableBucket now refuses a LANCE bucket declaring AES256 when the gateway reports no SSE-S3 key, naming s3.sse.kek and s3.sse.key; CreateVectorBucket returns it as ValidationException. ICEBERG buckets, whose declaration is only recorded, and tooling that sets no checker are unaffected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: refuse DeleteBucketEncryption on a bucket's declared encryption GetBucketEncryption reports a vector bucket's declared AES256, but DeleteBucketEncryption answered NoSuchBucketEncryptionConfiguration and the declaration kept applying. The declaration is fixed when the bucket is created, so the delete now returns MethodNotAllowed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): keep an inherited WEED_S3_SSE_KEK out of the test cluster The gateway refuses to start SSE-S3 with both s3.sse.kek and s3.sse.key set, so a developer shell exporting WEED_S3_SSE_KEK made the AES256 bucket test fail. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- test/s3vectors/conftest.py | 4 +- test/s3vectors/s3vectors_test.py | 30 +++++++- weed/command/s3.go | 1 + weed/s3api/s3_bucket_encryption.go | 22 +++++- weed/s3api/s3_sse_s3.go | 7 ++ weed/s3api/s3api_bucket_config.go | 5 ++ .../s3api_bucket_default_encryption_test.go | 71 +++++++++++++++++++ weed/s3api/s3api_tables.go | 6 ++ weed/s3api/s3tables/handler.go | 8 +++ weed/s3api/s3tables/handler_bucket_create.go | 6 ++ .../s3tables/handler_bucket_format_test.go | 38 ++++++++++ weed/s3api/s3tables/manager.go | 5 ++ weed/s3api/s3tables/utils.go | 19 +++++ weed/s3api/s3vectors/handlers_test.go | 26 +++++++ weed/s3api/s3vectors/server.go | 4 ++ 15 files changed, 249 insertions(+), 3 deletions(-) diff --git a/test/s3vectors/conftest.py b/test/s3vectors/conftest.py index 69f4831bba..9c99a758e7 100644 --- a/test/s3vectors/conftest.py +++ b/test/s3vectors/conftest.py @@ -102,7 +102,9 @@ def run_cluster(param, tmp_path_factory, identities): def start(name, args, unset=()): log = stack.enter_context((directory / f"{name}.log").open("w")) env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test", - AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true") + AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true", + WEED_S3_SSE_KEY="s3vectors-test") + env.pop("WEED_S3_SSE_KEK", None) for key in unset: env.pop(key) process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env) diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index adbeb1c246..51ebc4b5b4 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -1,5 +1,6 @@ """Run the S3 Vectors API against Go/Rust volumes and single/distributed queries.""" +import contextlib import json import time import uuid @@ -297,17 +298,44 @@ def test_vector_policy_permissions(vectors, vector_clients, index): vectors.delete_index(**other) +def index_object_encryption(vectors, bucket): + s3 = boto3.client("s3", endpoint_url=vectors.meta.endpoint_url, aws_access_key_id="test", + aws_secret_access_key="test", region_name="us-east-1") + keys = [o["Key"] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=f"vector/{INDEX}/") + for o in page.get("Contents", []) if o["Size"] > 0] + assert any("/data/" in key for key in keys) and any("/_versions/" in key for key in keys) + return {key: s3.head_object(Bucket=bucket, Key=key).get("ServerSideEncryption") for key in keys} + + def test_vector_bucket_encryption(vectors): bucket = "encryption-" + uuid.uuid4().hex[:16] + plain = bucket + "-plain" config = {"sseType": "AES256"} vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config) + vectors.create_vector_bucket(vectorBucketName=plain) try: assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms", encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"}) assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == [] + + for name in (bucket, plain): + vectors.create_index(vectorBucketName=name, indexName=INDEX, dataType="float32", dimension=4, + distanceMetric="euclidean") + vectors.put_vectors(vectorBucketName=name, indexName=INDEX, vectors=[ + {"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"color": "red"}}, + {"key": "b", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"color": "blue"}}, + ]) + hits = vectors.query_vectors(vectorBucketName=name, indexName=INDEX, queryVector={"float32": [1, 0, 0, 0]}, + topK=1, returnMetadata=True)["vectors"] + assert [(h["key"], h["metadata"]) for h in hits] == [("a", {"color": "red"})] + assert set(index_object_encryption(vectors, bucket).values()) == {"AES256"} + assert set(index_object_encryption(vectors, plain).values()) == {None} finally: - vectors.delete_vector_bucket(vectorBucketName=bucket) + for name in (bucket, plain): + with contextlib.suppress(ClientError): + vectors.delete_index(vectorBucketName=name, indexName=INDEX) + vectors.delete_vector_bucket(vectorBucketName=name) def test_catalog_pagination(vectors, index): diff --git a/weed/command/s3.go b/weed/command/s3.go index 94ed45d9f0..29e9a23f2d 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -724,6 +724,7 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend) vectorsServer.SetRegion(s3tables.DefaultRegion) vectorsServer.SetAccountID(s3tables.DefaultAccountID) + vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured) vectorsServer.RegisterRoutes(vectorsRouter) return vectorsRouter } diff --git a/weed/s3api/s3_bucket_encryption.go b/weed/s3api/s3_bucket_encryption.go index a1e647750c..b5ba8c6b54 100644 --- a/weed/s3api/s3_bucket_encryption.go +++ b/weed/s3api/s3_bucket_encryption.go @@ -213,12 +213,28 @@ func (s3a *S3ApiServer) getEncryptionConfiguration(bucket string) (*s3_pb.Encryp } if metadata.Encryption == nil { - return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + return s3a.declaredEncryption(bucket) } return metadata.Encryption, s3err.ErrNone } +// declaredEncryption is the encryption a table bucket's record declares. It +// applies while the bucket has no S3 default encryption of its own. +func (s3a *S3ApiServer) declaredEncryption(bucket string) (*s3_pb.EncryptionConfiguration, s3err.ErrorCode) { + if s3a.bucketConfigCache == nil { + return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + } + config, errCode := s3a.getBucketConfig(bucket) + if errCode != s3err.ErrNone { + return nil, errCode + } + if config.DeclaredEncryption == nil { + return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + } + return config.DeclaredEncryption, s3err.ErrNone +} + // updateEncryptionConfiguration updates the encryption configuration for a bucket func (s3a *S3ApiServer) updateEncryptionConfiguration(bucket string, encryptionConfig *s3_pb.EncryptionConfiguration) s3err.ErrorCode { // Update using structured API @@ -249,6 +265,10 @@ func (s3a *S3ApiServer) removeEncryptionConfiguration(bucket string) s3err.Error } if metadata.Encryption == nil { + // A bucket's declared encryption is fixed when the bucket is created. + if declared, _ := s3a.declaredEncryption(bucket); declared != nil { + return s3err.ErrMethodNotAllowed + } return s3err.ErrNoSuchBucketEncryptionConfiguration } diff --git a/weed/s3api/s3_sse_s3.go b/weed/s3api/s3_sse_s3.go index 8ce3542d28..32c386e428 100644 --- a/weed/s3api/s3_sse_s3.go +++ b/weed/s3api/s3_sse_s3.go @@ -778,6 +778,13 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) { // The DEK is encrypted with the super key and stored in object metadata } +// IsConfigured reports whether a KEK is loaded, i.e. SSE-S3 can encrypt. +func (km *SSES3KeyManager) IsConfigured() bool { + km.mu.RLock() + defer km.mu.RUnlock() + return len(km.superKey) > 0 +} + // GetMasterKey returns a derived key from the master KEK for STS signing // This uses HKDF to isolate the STS security domain from the SSE-S3 domain func (km *SSES3KeyManager) GetMasterKey() []byte { diff --git a/weed/s3api/s3api_bucket_config.go b/weed/s3api/s3api_bucket_config.go index 254b2adb83..427e58dcf5 100644 --- a/weed/s3api/s3api_bucket_config.go +++ b/weed/s3api/s3api_bucket_config.go @@ -24,6 +24,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // BucketConfig represents cached bucket configuration. Only fields parsed @@ -56,6 +57,7 @@ type BucketConfig struct { LifecycleTransitionMinSize string Tags map[string]string // parsed from entry content Encryption *s3_pb.EncryptionConfiguration // parsed from entry content + DeclaredEncryption *s3_pb.EncryptionConfiguration // declared by a table bucket's record KMSKeyCache *BucketKMSCache // Per-bucket KMS key cache for SSE-KMS operations LastModified time.Time } @@ -441,6 +443,9 @@ func (s3a *S3ApiServer) newBucketConfigFromEntry(bucket string, entry *filer_pb. config.ObjectLockConfig = objectLockConfig } config.BucketPolicy = loadBucketPolicyFromExtended(entry, bucket) + if algorithm := s3tables.DeclaredBucketEncryption(entry); algorithm != "" { + config.DeclaredEncryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: algorithm} + } if lifecycleXML, exists := entry.Extended[bucketLifecycleConfigurationXMLKey]; exists && len(lifecycleXML) > 0 { config.LifecycleXML = lifecycleXML diff --git a/weed/s3api/s3api_bucket_default_encryption_test.go b/weed/s3api/s3api_bucket_default_encryption_test.go index 74484593f9..1ac7b19447 100644 --- a/weed/s3api/s3api_bucket_default_encryption_test.go +++ b/weed/s3api/s3api_bucket_default_encryption_test.go @@ -2,6 +2,7 @@ package s3api import ( "bytes" + "errors" "net/http" "net/http/httptest" "strings" @@ -12,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // Issue 11647: Bucket default encryption (SSE-S3) must be applied when @@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) { } } } + +// A vector bucket is a LANCE table bucket; the AES256 its record declares is +// the bucket's default encryption. An ICEBERG bucket's declaration is not +// applied, because its catalog reads client manifests straight from the filer. +func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) { + km := GetSSES3KeyManager() + oldSuperKey := km.superKey + km.superKey = make([]byte, 32) + for i := range km.superKey { + km.superKey[i] = byte(i + 1) + } + t.Cleanup(func() { + km.superKey = oldSuperKey + }) + + volume := startFakeVolumeServer(t) + filerImpl := &ambiguousPutFiler{ + volume: volume, + entries: map[string]*filer_pb.Entry{}, + apply: true, + } + s3a := newPutTestServer(t, startFakeFiler(t, filerImpl)) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} { + s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{ + Name: bucket, + Extended: map[string][]byte{ + s3tables.ExtendedKeyTableBucket: []byte("true"), + s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format + + `","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`), + }, + })) + } + + cfg, err := s3a.GetBucketEncryptionConfig("lance") + if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 { + t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err) + } + if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil { + t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err) + } + if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) { + t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err) + } + if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed { + t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code) + } + if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration { + t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code) + } + + for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} { + filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance" + r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil) + _, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "") + if code != s3err.ErrNone { + t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone) + } + entry := filerImpl.entries[filePath] + if entry == nil || len(entry.Chunks) == 0 { + t.Fatalf("no entry with chunks at %s", filePath) + } + for i, chunk := range entry.Chunks { + if chunk.SseType != want { + t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want) + } + } + } +} diff --git a/weed/s3api/s3api_tables.go b/weed/s3api/s3api_tables.go index 5a382d0f3f..6b50017197 100644 --- a/weed/s3api/s3api_tables.go +++ b/weed/s3api/s3api_tables.go @@ -58,6 +58,11 @@ func (st *S3TablesApiServer) SetLocalAccountChecker(fn func(string) bool) { st.handler.SetLocalAccountChecker(fn) } +// SetEncryptionChecker sets the handler's SSE-S3 availability predicate. +func (st *S3TablesApiServer) SetEncryptionChecker(fn func() bool) { + st.handler.SetEncryptionChecker(fn) +} + // S3TablesHandler handles S3 Tables API requests func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) { st.handler.HandleRequest(w, r, st) @@ -72,6 +77,7 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) { // Create S3 Tables handler s3TablesApi := NewS3TablesApiServer(s3a) + s3TablesApi.SetEncryptionChecker(GetSSES3KeyManager().IsConfigured) if s3a.iam != nil { s3TablesApi.SetLocalAccountChecker(s3a.iam.isConfiguredAccountID) } diff --git a/weed/s3api/s3tables/handler.go b/weed/s3api/s3tables/handler.go index 2666480e40..8e1b932bac 100644 --- a/weed/s3api/s3tables/handler.go +++ b/weed/s3api/s3tables/handler.go @@ -76,6 +76,7 @@ type S3TablesHandler struct { trusted bool // Trusted local tooling (shell/admin) bypasses authorization iamAuthorizer IAMAuthorizer localAccountIDs func(string) bool + canEncrypt func() bool } // NewS3TablesHandler creates a new S3 Tables handler @@ -119,6 +120,13 @@ func (h *S3TablesHandler) SetLocalAccountChecker(fn func(string) bool) { h.localAccountIDs = fn } +// SetEncryptionChecker injects the predicate that tells whether the gateway +// holds an SSE-S3 key, so a bucket whose declared AES256 the write path +// applies is not created where every write would then fail. +func (h *S3TablesHandler) SetEncryptionChecker(fn func() bool) { + h.canEncrypt = fn +} + // FilerClient interface for filer operations type FilerClient interface { WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error diff --git a/weed/s3api/s3tables/handler_bucket_create.go b/weed/s3api/s3tables/handler_bucket_create.go index 5d6464c0e1..0370354798 100644 --- a/weed/s3api/s3tables/handler_bucket_create.go +++ b/weed/s3api/s3tables/handler_bucket_create.go @@ -80,6 +80,12 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http } } + if len(req.EncryptionConfiguration) > 0 && IsCatalogOnlyFormat(bucketFormat) && h.canEncrypt != nil && !h.canEncrypt() { + msg := "AES256 encryption needs an SSE-S3 key on the S3 gateway: set s3.sse.kek or s3.sse.key in security.toml" + h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, msg) + return errors.New(msg) + } + bucketPath := GetTableBucketPath(req.Name) // Check if bucket already exists and ensure no conflict with object store buckets diff --git a/weed/s3api/s3tables/handler_bucket_format_test.go b/weed/s3api/s3tables/handler_bucket_format_test.go index b024314a76..7787b8c913 100644 --- a/weed/s3api/s3tables/handler_bucket_format_test.go +++ b/weed/s3api/s3tables/handler_bucket_format_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "testing" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" "github.com/stretchr/testify/require" @@ -107,3 +109,39 @@ func TestNormalizeFormat(t *testing.T) { } } } + +func TestDeclaredBucketEncryption(t *testing.T) { + entry := func(format string, cfg map[string]interface{}) *filer_pb.Entry { + meta, _ := json.Marshal(tableBucketMetadata{Name: "b", Format: format, EncryptionConfiguration: cfg}) + return &filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyTableBucket: []byte("true"), ExtendedKeyMetadata: meta}} + } + aes := map[string]interface{}{"sseAlgorithm": "AES256"} + + require.Equal(t, "AES256", DeclaredBucketEncryption(entry(FormatLance, aes))) + require.Empty(t, DeclaredBucketEncryption(entry(FormatLance, nil))) + require.Empty(t, DeclaredBucketEncryption(entry(FormatIceberg, aes)), "the Iceberg catalog reads client manifests raw") + require.Empty(t, DeclaredBucketEncryption(entry("", aes)), "a bucket without a format is Iceberg") + require.Empty(t, DeclaredBucketEncryption(&filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyMetadata: entry(FormatLance, aes).Extended[ExtendedKeyMetadata]}}), "not a table bucket") + require.Empty(t, DeclaredBucketEncryption(nil)) +} + +// Only a format whose files the catalog leaves to clients gets its declared +// AES256 applied, so only that format needs the gateway's SSE-S3 key. +func TestCreateTableBucketNeedsSSES3ForADeclaredLanceBucket(t *testing.T) { + fs := s3tablestest.Start(t) + m := NewManager() + m.SetEncryptionChecker(func() bool { return false }) + create := func(name, format string, cfg map[string]interface{}) error { + return m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateTableBucket", + &CreateTableBucketRequest{Name: name, Format: format, EncryptionConfiguration: cfg}, nil, s3_constants.AccountAdminId) + } + aes := map[string]interface{}{"sseAlgorithm": "AES256"} + + err := create("lance-sealed", FormatLance, aes) + var s3Err *S3TablesError + require.ErrorAs(t, err, &s3Err) + require.Equal(t, ErrCodeInvalidRequest, s3Err.Type) + require.Contains(t, s3Err.Message, "s3.sse.kek") + require.NoError(t, create("lance-plain", FormatLance, nil)) + require.NoError(t, create("iceberg-sealed", FormatIceberg, aes), "an Iceberg bucket only records the declaration") +} diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index 0d6cb441e1..bffe3c9c5f 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -46,6 +46,11 @@ func (m *Manager) SetTrusted(trusted bool) { m.handler.SetTrusted(trusted) } +// SetEncryptionChecker sets the handler's SSE-S3 availability predicate. +func (m *Manager) SetEncryptionChecker(fn func() bool) { + m.handler.SetEncryptionChecker(fn) +} + // PrincipalFor resolves the caller of r the way the catalog's handlers do, // so it matches the owner CreateTableBucket records. A nil request is the // handler's account ID. diff --git a/weed/s3api/s3tables/utils.go b/weed/s3api/s3tables/utils.go index f87e34e3fc..7837ed9cbc 100644 --- a/weed/s3api/s3tables/utils.go +++ b/weed/s3api/s3tables/utils.go @@ -3,6 +3,7 @@ package s3tables import ( "crypto/rand" "encoding/hex" + "encoding/json" "fmt" "net/url" "path" @@ -192,6 +193,24 @@ func ValidateEncryptionConfiguration(cfg map[string]interface{}, sseField string return nil } +// DeclaredBucketEncryption returns the sseAlgorithm a table bucket's record +// declares, for the S3 write path to apply, when the bucket's format leaves +// every file to its clients. An ICEBERG bucket gets "": its catalog reads the +// manifests clients upload straight from the filer, past S3 decryption. +func DeclaredBucketEncryption(entry *filer_pb.Entry) string { + if !IsTableBucketEntry(entry) { + return "" + } + var metadata tableBucketMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil || !IsCatalogOnlyFormat(metadata.Format) { + return "" + } + if algorithm, _ := metadata.EncryptionConfiguration["sseAlgorithm"].(string); algorithm == "AES256" { + return algorithm + } + return "" +} + func parseS3Location(location string) (bucket, tablePath string, err error) { if !strings.HasPrefix(location, "s3://") { return "", "", fmt.Errorf("unsupported location: %s", location) diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index 76849850f6..bb31b644a7 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -704,6 +704,32 @@ func TestValidateEncryptionConfigurationRejectsCustomerKeys(t *testing.T) { } } +// The gateway applies a vector bucket's AES256 to every index write, so a +// gateway without an SSE-S3 key refuses the bucket instead of failing each +// later write. +func TestCreateVectorBucketNeedsSSES3ForAES256(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + keyLoaded := false + s.SetEncryptionChecker(func() bool { return keyLoaded }) + aes := map[string]interface{}{"sseType": "AES256"} + + w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes}) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") || !strings.Contains(w.Body.String(), "s3.sse.key") { + t.Fatalf("AES256 without a key: %d %s, want 400 ValidationException naming s3.sse.key", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "sealed"}); w.Code != http.StatusNotFound { + t.Fatalf("refused bucket exists: %d %s", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "plain"}); w.Code != http.StatusOK { + t.Fatalf("unencrypted bucket without a key: %d %s", w.Code, w.Body.String()) + } + keyLoaded = true + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes}); w.Code != http.StatusOK { + t.Fatalf("AES256 with a key: %d %s", w.Code, w.Body.String()) + } +} + // A vector index is colocated: its catalog entry sits at the dataset root, so // the catalog delete removes entry and data together. func TestDeleteIndexRemovesDatasetDirectory(t *testing.T) { diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index af77da9b84..c9be580589 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -65,6 +65,10 @@ func (s *Server) SetRegion(region string) { s.region = region } // SetAccountID configures the account id embedded in the ARNs this server mints. func (s *Server) SetAccountID(id string) { s.accountID = id } +// SetEncryptionChecker tells CreateVectorBucket whether the gateway can +// apply the AES256 a bucket declares. +func (s *Server) SetEncryptionChecker(fn func() bool) { s.internal.SetEncryptionChecker(fn) } + // RegisterRoutes mounts POST / dispatching on the X-Amz-Target header and // POST /{Operation} for clients that route per-path. Metrics wrap Auth like the // S3 track middleware so rejected requests still count toward the operation. From 02d22800aaac1d8dac541ffbd32c844562be2ae8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 00:18:44 -0800 Subject: [PATCH 17/31] s3vectors: accept bodies up to the 32 MiB limit, report larger ones as a size error, log engine failures (#12320) * s3api: hash an s3vectors body up to the Vectors API limit SigV4 verification of a non-S3 request hashed at most 10 MiB of the body and silently dropped the rest, so any S3 Vectors request over 10 MiB failed its signature check with 403, although the API accepts 32 MiB. The limit now follows the signing service, and a body over it is reported as ErrEntityTooLarge instead of being truncated; the embedded IAM answers that with InvalidInput, as it already does for a body over its own limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3vectors: answer an oversized request with a ValidationException A body over 32 MiB was refused as "403 AccessDenied: authentication failed". A declared Content-Length over the limit is now refused before authentication, and a streamed body over it, whether the signature check or the handler reads it, gets the same "request body exceeds 32 MiB" ValidationException. The limit is the one SigV4 verification uses. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3vectors: log engine and catalog failures at the default level An engine error reached the client as a bare "internal error", and the cause was logged only with -v=1, so a misconfigured engine left nothing in a default log. Engine and catalog failures are now logged as errors, and a reported unavailability as a warning, with the operation, bucket and index. The response still leaves the engine's message out, since it can name internal hosts and paths, and points at the gateway log instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: hash the signed body through readRequestBody streamHashRequestBody repeated readRequestBody's read-past-the-limit check; it now reads the body with readRequestBody and hashes the bytes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- weed/s3api/auth_signature_v4.go | 25 +++++---- weed/s3api/auto_signature_v4_test.go | 78 +++++++++++++++++---------- weed/s3api/s3_constants/s3_config.go | 3 ++ weed/s3api/s3api_embedded_iam.go | 2 + weed/s3api/s3api_embedded_iam_test.go | 1 + weed/s3api/s3vectors/auth_test.go | 58 ++++++++++++++++++++ weed/s3api/s3vectors/handlers.go | 26 +++++---- weed/s3api/s3vectors/handlers_test.go | 30 +++++++++++ weed/s3api/s3vectors/server.go | 11 +++- weed/s3api/s3vectors/types.go | 15 +++++- 10 files changed, 197 insertions(+), 52 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 661bd7002f..2332d36a3a 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -25,6 +25,7 @@ import ( "crypto/subtle" "encoding/base64" "encoding/hex" + "errors" "io" "net" "net/http" @@ -74,22 +75,19 @@ func streamHashRequestBody(r *http.Request, sizeLimit int64) (string, error) { return emptySHA256, nil } - limitedReader := io.LimitReader(r.Body, sizeLimit) - hasher := sha256.New() - var bodyBuffer bytes.Buffer - - // Use io.Copy with an io.MultiWriter to hash and buffer the body simultaneously. - if _, err := io.Copy(io.MultiWriter(hasher, &bodyBuffer), limitedReader); err != nil { + body, err := readRequestBody(r, sizeLimit) + if err != nil { return "", err } - r.Body = io.NopCloser(bytes.NewReader(bodyBuffer.Bytes())) + r.Body = io.NopCloser(bytes.NewReader(body)) - if bodyBuffer.Len() == 0 { + if len(body) == 0 { return emptySHA256, nil } - return hex.EncodeToString(hasher.Sum(nil)), nil + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil } // getContentSha256Cksum retrieves the "x-amz-content-sha256" header value. @@ -655,8 +653,15 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) hashedPayload := getContentSha256Cksum(r) if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { + sizeLimit := int64(iamRequestBodyLimit) + if signV4Values.Credential.scope.service == "s3vectors" { + sizeLimit = s3_constants.S3VectorsMaxRequestBodyBytes + } var hashErr error - hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit) + hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit) + if errors.Is(hashErr, errRequestBodyTooLarge) { + return nil, s3err.ErrEntityTooLarge + } if hashErr != nil { return nil, s3err.ErrInternalError } diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index e7747515d6..34874ae8dc 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -16,6 +16,8 @@ import ( "time" "unicode/utf8" + "github.com/aws/aws-sdk-go/aws/credentials" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" @@ -1820,18 +1822,10 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) { ", SignedHeaders=content-type;host;x-amz-date, Signature=dummysignature" req.Header.Set("Authorization", authHeader) - // The function should complete successfully but limit the body to 10 MiB identity, _, errCode := iam.doesSignatureMatch(req) - // Should get signature mismatch (dummy signature) but not internal error - assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode) + assert.Equal(t, s3err.ErrEntityTooLarge, errCode) assert.Nil(t, identity) - - // Verify the body was truncated to the limit (10 MiB) - bodyBytes, err := io.ReadAll(req.Body) - assert.NoError(t, err) - assert.Equal(t, 10*(1<<20), len(bodyBytes)) // Should be exactly 10 MiB - assert.Equal(t, strings.Repeat("A", 10*(1<<20)), string(bodyBytes)) // All As, but truncated } // Test the streaming hash implementation directly @@ -1922,30 +1916,58 @@ func TestStreamingVsNonStreamingConsistency(t *testing.T) { } } -// Test streaming with size limit enforcement func TestStreamingWithSizeLimit(t *testing.T) { - // Create a payload larger than the limit - largePayload := strings.Repeat("A", 11*(1<<20)) // 11 MiB - - req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(largePayload)) + exact := strings.Repeat("A", 1<<20) + req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact)) assert.NoError(t, err) - - // Stream with the limit - hash, err := streamHashRequestBody(req, iamRequestBodyLimit) + hash, err := streamHashRequestBody(req, 1<<20) assert.NoError(t, err) + assert.Equal(t, getSHA256Hash([]byte(exact)), hash) - // Verify the hash is computed for the truncated content (10 MiB) - truncatedPayload := strings.Repeat("A", 10*(1<<20)) - expectedHash := sha256.Sum256([]byte(truncatedPayload)) - expectedHashStr := hex.EncodeToString(expectedHash[:]) - - assert.Equal(t, expectedHashStr, hash) - - // Verify the body was truncated - bodyBytes, err := io.ReadAll(req.Body) + req, err = http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact+"A")) assert.NoError(t, err) - assert.Equal(t, 10*(1<<20), len(bodyBytes)) - assert.Equal(t, truncatedPayload, string(bodyBytes)) + _, err = streamHashRequestBody(req, 1<<20) + assert.ErrorIs(t, err, errRequestBodyTooLarge) +} + +func TestNonS3BodyHashLimitFollowsService(t *testing.T) { + iam := &IdentityAccessManagement{} + assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{ + Name: "vec", + Credentials: []*iam_pb.Credential{{AccessKey: "vec_key", SecretKey: "vec_secret"}}, + Actions: []string{"Admin"}, + }}, + })) + signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", "")) + + for _, tc := range []struct { + service string + size int + want s3err.ErrorCode + }{ + {"s3vectors", 20 << 20, s3err.ErrNone}, + {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, + {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, + {"iam", iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + } { + t.Run(fmt.Sprintf("%s_%d", tc.service, tc.size), func(t *testing.T) { + body := strings.Repeat("v", tc.size) + req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body)) + assert.NoError(t, err) + _, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now()) + assert.NoError(t, err) + + identity, _, errCode := iam.doesSignatureMatch(req) + assert.Equal(t, tc.want, errCode) + if tc.want == s3err.ErrNone { + assert.Equal(t, "vec", identity.Name) + got, err := io.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, len(body), len(got)) + } + }) + } } // Benchmark streaming vs non-streaming memory usage diff --git a/weed/s3api/s3_constants/s3_config.go b/weed/s3api/s3_constants/s3_config.go index d2d2c257aa..ab34ce39af 100644 --- a/weed/s3api/s3_constants/s3_config.go +++ b/weed/s3api/s3_constants/s3_config.go @@ -13,6 +13,9 @@ var ( Separator = ":" ) +// S3VectorsMaxRequestBodyBytes caps the body of an S3 Vectors API request. +const S3VectorsMaxRequestBodyBytes = 32 << 20 + func Concat(elements ...string) string { return strings.Join(elements, Separator) } diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index 1c6aa1bec0..6b5e244469 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -342,6 +342,8 @@ func iamAuthErrorCode(errCode s3err.ErrorCode) string { return "RequestTimeTooSkewed" case s3err.ErrInternalError: return iam.ErrCodeServiceFailureException + case s3err.ErrEntityTooLarge: + return "InvalidInput" default: return "IncompleteSignature" } diff --git a/weed/s3api/s3api_embedded_iam_test.go b/weed/s3api/s3api_embedded_iam_test.go index 4269958de9..a7043b7114 100644 --- a/weed/s3api/s3api_embedded_iam_test.go +++ b/weed/s3api/s3api_embedded_iam_test.go @@ -3041,5 +3041,6 @@ func TestIamAuthErrorCode(t *testing.T) { assert.Equal(t, "SignatureDoesNotMatch", iamAuthErrorCode(s3err.ErrSignatureDoesNotMatch)) assert.Equal(t, "AccessDenied", iamAuthErrorCode(s3err.ErrAccessDenied)) assert.Equal(t, "RequestTimeTooSkewed", iamAuthErrorCode(s3err.ErrRequestTimeTooSkewed)) + assert.Equal(t, "InvalidInput", iamAuthErrorCode(s3err.ErrEntityTooLarge)) assert.Equal(t, "IncompleteSignature", iamAuthErrorCode(s3err.ErrMissingFields)) } diff --git a/weed/s3api/s3vectors/auth_test.go b/weed/s3api/s3vectors/auth_test.go index a333265d0d..56e326e5a9 100644 --- a/weed/s3api/s3vectors/auth_test.go +++ b/weed/s3api/s3vectors/auth_test.go @@ -1,8 +1,11 @@ package s3vectors import ( + "errors" + "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api" @@ -12,9 +15,15 @@ import ( type fakeAuthenticator struct { identity *s3api.Identity + errCode s3err.ErrorCode + calls int } func (f *fakeAuthenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { + f.calls++ + if f.errCode != s3err.ErrNone { + return "", nil, f.errCode + } return f.identity.Name, f.identity, s3err.ErrNone } @@ -35,3 +44,52 @@ func TestAuthKeepsIdentityObject(t *testing.T) { t.Fatalf("identity object missing from context: %v", got) } } + +func TestAuthAnswersOversizedBodyWithValidationException(t *testing.T) { + alice := &s3api.Identity{Name: "alice", Actions: []s3api.Action{"Admin"}} + for _, tc := range []struct { + name string + contentLength int64 + errCode s3err.ErrorCode + wantStatus int + wantCode string + wantAuthCalls int + }{ + {"declared length over the limit", maxRequestBodyBytes + 1, s3err.ErrNone, http.StatusBadRequest, "ValidationException", 0}, + {"streamed body over the limit", -1, s3err.ErrEntityTooLarge, http.StatusBadRequest, "ValidationException", 1}, + {"bad signature", 10, s3err.ErrSignatureDoesNotMatch, http.StatusForbidden, "AccessDenied", 1}, + {"declared length at the limit", maxRequestBodyBytes, s3err.ErrNone, http.StatusOK, "", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + auth := &fakeAuthenticator{identity: alice, errCode: tc.errCode} + s := &Server{authenticator: auth} + req := httptest.NewRequest(http.MethodPost, "/PutVectors", strings.NewReader("{}")) + req.ContentLength = tc.contentLength + rec := httptest.NewRecorder() + s.Auth(func(w http.ResponseWriter, r *http.Request) {})(rec, req) + + if rec.Code != tc.wantStatus || auth.calls != tc.wantAuthCalls { + t.Fatalf("status %d after %d auth calls, want %d after %d: %s", rec.Code, auth.calls, tc.wantStatus, tc.wantAuthCalls, rec.Body) + } + if tc.wantCode != "" && !strings.Contains(rec.Body.String(), `"code":"`+tc.wantCode+`"`) { + t.Fatalf("body %s, want code %s", rec.Body, tc.wantCode) + } + if tc.wantCode == "ValidationException" && !strings.Contains(rec.Body.String(), "request body exceeds 32 MiB") { + t.Fatalf("body %s does not name the limit", rec.Body) + } + }) + } +} + +func TestReadJSONRefusesBodyOverTheLimit(t *testing.T) { + var req PutVectorsRequest + over := httptest.NewRequest(http.MethodPost, "/PutVectors", io.MultiReader( + strings.NewReader(`{"indexName":"`), strings.NewReader(strings.Repeat("a", maxRequestBodyBytes)), strings.NewReader(`"}`))) + if err := readJSON(over, &req); !errors.Is(err, errRequestBodyTooLarge) { + t.Fatalf("readJSON over the limit = %v, want %v", err, errRequestBodyTooLarge) + } + within := httptest.NewRequest(http.MethodPost, "/PutVectors", strings.NewReader(`{"indexName":"idx"}`)) + if err := readJSON(within, &req); err != nil || req.IndexName != "idx" { + t.Fatalf("readJSON within the limit = %v, indexName %q", err, req.IndexName) + } +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 593e4237ce..7e7bd12e83 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -183,7 +183,8 @@ func catalogErrCode(err error) string { func catalogErr(w http.ResponseWriter, err error) { var apiErr *s3tables.S3TablesError if !errors.As(err, &apiErr) { - writeError(w, http.StatusInternalServerError, "InternalFailure", "internal error") + glog.Errorf("S3 Vectors: catalog: %v", err) + writeError(w, http.StatusInternalServerError, "InternalFailure", internalErrorMessage) return } switch { @@ -201,10 +202,12 @@ func catalogErr(w http.ResponseWriter, err error) { } } +const internalErrorMessage = "internal error, see the S3 gateway log" + // vectorErr maps a VectorService gRPC status onto the API's error shape. // Engine messages reach clients only for codes that describe the request // itself; transport and internal details stay in the server log. -func vectorErr(w http.ResponseWriter, err error) { +func vectorErr(w http.ResponseWriter, r *http.Request, arns vectorBucketARNs, err error) { msg := status.Convert(err).Message() switch status.Code(err) { case codes.NotFound: @@ -214,10 +217,11 @@ func vectorErr(w http.ResponseWriter, err error) { case codes.PermissionDenied, codes.Unauthenticated: writeError(w, http.StatusForbidden, "AccessDeniedException", "access denied") case codes.Unavailable: + glog.Warningf("S3 Vectors: %s %s/%s: %v", operationFromRequest(r), arns.bucket, arns.index, err) writeError(w, http.StatusServiceUnavailable, "ServiceUnavailableException", "vector service unavailable") default: - glog.V(1).Infof("S3 Vectors: engine error: %v", err) - writeError(w, http.StatusInternalServerError, "InternalFailure", "internal error") + glog.Errorf("S3 Vectors: %s %s/%s: %v", operationFromRequest(r), arns.bucket, arns.index, err) + writeError(w, http.StatusInternalServerError, "InternalFailure", internalErrorMessage) } } @@ -1026,7 +1030,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { } else if gerr != nil { glog.Errorf("S3 Vectors: rollback lookup for %s/%s failed: %v", arns.bucket, arns.index, gerr) } - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &CreateIndexResponse{IndexArn: s.indexArn(arns.bucket, arns.index)}) @@ -1279,7 +1283,7 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) { Vectors: vectors, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &PutVectorsResponse{}) @@ -1360,7 +1364,7 @@ func (s *Server) handleGetVectors(w http.ResponseWriter, r *http.Request) { ReturnMetadata: req.ReturnMetadata, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &GetVectorsResponse{ @@ -1419,7 +1423,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { } resp, err := s.vectors.ListVectors(r.Context(), listReq) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } // An engine that predates segment_count lists every row; page through @@ -1429,7 +1433,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { out, nextToken, err := s.listSegmentVectors(r.Context(), listReq.Table, offset, maxResults, uint64(*req.SegmentCount), uint64(*req.SegmentIndex), req.ReturnData, req.ReturnMetadata, resp) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &ListVectorsResponse{ @@ -1559,7 +1563,7 @@ func (s *Server) handleDeleteVectors(w http.ResponseWriter, r *http.Request) { Keys: req.Keys, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &DeleteVectorsResponse{}) @@ -1932,7 +1936,7 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { ReturnDistance: req.ReturnDistance, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } out := &QueryVectorsResponse{DistanceMetric: metric, Vectors: []*QueryOutputVector{}} diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index bb31b644a7..6ed043d8bc 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -19,6 +19,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // fakeVectors serves a fixed row set with the engine's offset-token paging @@ -1073,3 +1075,31 @@ func TestCreateIndexSurvivesAConcurrentNamespaceCreate(t *testing.T) { t.Fatalf("first create index: %d %s", w.Code, w.Body.String()) } } + +func TestVectorErrKeepsEngineDetailsOutOfTheResponse(t *testing.T) { + arns := vectorBucketARNs{bucket: "media", index: "movies"} + for _, tc := range []struct { + err error + wantStatus int + wantCode string + wantMessage string + }{ + {status.Error(codes.Internal, "declare: 500 from http://10.0.0.5:8888/lance/v1/table/media$vector$movies/declare"), + http.StatusInternalServerError, "InternalFailure", internalErrorMessage}, + {status.Error(codes.Unavailable, "no weed-vector registered on master"), + http.StatusServiceUnavailable, "ServiceUnavailableException", "vector service unavailable"}, + {status.Error(codes.InvalidArgument, "vector has dimension 2, index has 3"), + http.StatusBadRequest, "ValidationException", "vector has dimension 2, index has 3"}, + } { + rec := httptest.NewRecorder() + vectorErr(rec, httptest.NewRequest(http.MethodPost, "/PutVectors", nil), arns, tc.err) + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if rec.Code != tc.wantStatus || body["code"] != tc.wantCode || body["message"] != tc.wantMessage { + t.Errorf("%v: got %d %s %q, want %d %s %q", status.Code(tc.err), rec.Code, body["code"], body["message"], + tc.wantStatus, tc.wantCode, tc.wantMessage) + } + } +} diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index c9be580589..ceb7389643 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -129,8 +129,17 @@ func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) { // Auth authenticates the caller and puts the identity in the request context. func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength > maxRequestBodyBytes { + writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) + return + } identity, identityObj, errCode := s.authenticator.AuthenticateRequest(r) - if errCode != s3err.ErrNone { + switch errCode { + case s3err.ErrNone: + case s3err.ErrEntityTooLarge: + writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) + return + default: writeError(w, http.StatusForbidden, "AccessDenied", "authentication failed") return } diff --git a/weed/s3api/s3vectors/types.go b/weed/s3api/s3vectors/types.go index a9c5b96c51..0bbbaff6f6 100644 --- a/weed/s3api/s3vectors/types.go +++ b/weed/s3api/s3vectors/types.go @@ -1,8 +1,10 @@ package s3vectors import ( + "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "time" @@ -35,10 +37,19 @@ func writeError(w http.ResponseWriter, status int, code, message string) { }) } -const maxRequestBodyBytes = 32 << 20 +const maxRequestBodyBytes = s3_constants.S3VectorsMaxRequestBodyBytes + +var errRequestBodyTooLarge = fmt.Errorf("request body exceeds %d MiB", maxRequestBodyBytes>>20) func readJSON(r *http.Request, req interface{}) error { - return json.NewDecoder(io.LimitReader(r.Body, maxRequestBodyBytes)).Decode(req) + body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodyBytes+1)) + if err != nil { + return err + } + if len(body) > maxRequestBodyBytes { + return errRequestBodyTooLarge + } + return json.NewDecoder(bytes.NewReader(body)).Decode(req) } // Request/response shapes for the AWS S3 Vectors API (awsJson1_1). From d431c1b35a57b32b83818aab29ca7bb1120d6448 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 01:15:42 -0800 Subject: [PATCH 18/31] s3api: let the vectors route, not the credential scope, raise the body limit (#12326) SigV4 verification picked the 32 MiB S3 Vectors body limit from the credential scope, which is not verified until the body is hashed. A request claiming the s3vectors scope on any other route could make the gateway buffer 32 MiB before authentication instead of 10 MiB. The vectors handler now sets the limit on the request context, and every other route keeps 10 MiB. Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK Co-authored-by: Claude Opus 5.5 --- weed/s3api/auth_signature_v4.go | 5 +---- weed/s3api/auto_signature_v4_test.go | 25 ++++++++++++++++--------- weed/s3api/s3_constants/header.go | 15 +++++++++++++++ weed/s3api/s3vectors/auth_test.go | 11 ++++++++--- weed/s3api/s3vectors/server.go | 1 + 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 2332d36a3a..b86c14b42c 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -653,10 +653,7 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) hashedPayload := getContentSha256Cksum(r) if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { - sizeLimit := int64(iamRequestBodyLimit) - if signV4Values.Credential.scope.service == "s3vectors" { - sizeLimit = s3_constants.S3VectorsMaxRequestBodyBytes - } + sizeLimit := s3_constants.GetRequestBodyLimitFromContext(r, iamRequestBodyLimit) var hashErr error hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit) if errors.Is(hashErr, errRequestBodyTooLarge) { diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 34874ae8dc..3ba5b874e1 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -1930,7 +1930,7 @@ func TestStreamingWithSizeLimit(t *testing.T) { assert.ErrorIs(t, err, errRequestBodyTooLarge) } -func TestNonS3BodyHashLimitFollowsService(t *testing.T) { +func TestNonS3BodyHashLimitFollowsRoute(t *testing.T) { iam := &IdentityAccessManagement{} assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ Identities: []*iam_pb.Identity{{ @@ -1941,20 +1941,27 @@ func TestNonS3BodyHashLimitFollowsService(t *testing.T) { })) signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", "")) + vectorsRoute := int64(s3_constants.S3VectorsMaxRequestBodyBytes) for _, tc := range []struct { - service string - size int - want s3err.ErrorCode + service string + routeLimit int64 + size int + want s3err.ErrorCode }{ - {"s3vectors", 20 << 20, s3err.ErrNone}, - {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, - {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, - {"iam", iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + {"s3vectors", vectorsRoute, 20 << 20, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, + // An s3vectors scope alone, on another route, keeps the 10 MiB limit. + {"s3vectors", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + {"iam", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, } { - t.Run(fmt.Sprintf("%s_%d", tc.service, tc.size), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_route%d_%d", tc.service, tc.routeLimit, tc.size), func(t *testing.T) { body := strings.Repeat("v", tc.size) req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body)) assert.NoError(t, err) + if tc.routeLimit > 0 { + req = req.WithContext(s3_constants.SetRequestBodyLimitInContext(req.Context(), tc.routeLimit)) + } _, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now()) assert.NoError(t, err) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index a49d40220e..6256d4376b 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -359,6 +359,7 @@ const ( contextKeyOriginalRequestPath contextKey = "s3-original-request-path" contextKeyAccessKeyId contextKey = "s3-access-key-id" contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id" + contextKeyRequestBodyLimit contextKey = "s3-request-body-limit" ) // identityHolder is a mutable container for the authenticated identity name, @@ -529,6 +530,20 @@ func GetOriginalRequestPathFromContext(r *http.Request) string { return "" } +// SetRequestBodyLimitInContext sets how much of the body SigV4 verification +// reads. The route sets it: the credential scope is not verified yet. +func SetRequestBodyLimitInContext(ctx context.Context, limit int64) context.Context { + return context.WithValue(ctx, contextKeyRequestBodyLimit, limit) +} + +// GetRequestBodyLimitFromContext returns the route's limit, or def. +func GetRequestBodyLimitFromContext(r *http.Request, def int64) int64 { + if limit, ok := r.Context().Value(contextKeyRequestBodyLimit).(int64); ok { + return limit + } + return def +} + // SetAccessKeyIdInContext stores the access key id the request authenticated // with: a static key's id, or an STS session's temporary key id. Several keys // can belong to one identity, so the identity name alone cannot tell their diff --git a/weed/s3api/s3vectors/auth_test.go b/weed/s3api/s3vectors/auth_test.go index 56e326e5a9..a7e1fb88b7 100644 --- a/weed/s3api/s3vectors/auth_test.go +++ b/weed/s3api/s3vectors/auth_test.go @@ -14,13 +14,15 @@ import ( ) type fakeAuthenticator struct { - identity *s3api.Identity - errCode s3err.ErrorCode - calls int + identity *s3api.Identity + errCode s3err.ErrorCode + calls int + bodyLimit int64 } func (f *fakeAuthenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { f.calls++ + f.bodyLimit = s3_constants.GetRequestBodyLimitFromContext(r, 0) if f.errCode != s3err.ErrNone { return "", nil, f.errCode } @@ -71,6 +73,9 @@ func TestAuthAnswersOversizedBodyWithValidationException(t *testing.T) { if rec.Code != tc.wantStatus || auth.calls != tc.wantAuthCalls { t.Fatalf("status %d after %d auth calls, want %d after %d: %s", rec.Code, auth.calls, tc.wantStatus, tc.wantAuthCalls, rec.Body) } + if auth.calls > 0 && auth.bodyLimit != maxRequestBodyBytes { + t.Fatalf("signature check reads up to %d bytes, want the route's %d", auth.bodyLimit, maxRequestBodyBytes) + } if tc.wantCode != "" && !strings.Contains(rec.Body.String(), `"code":"`+tc.wantCode+`"`) { t.Fatalf("body %s, want code %s", rec.Body, tc.wantCode) } diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index ceb7389643..c2bb46a607 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -133,6 +133,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) return } + r = r.WithContext(s3_constants.SetRequestBodyLimitInContext(r.Context(), maxRequestBodyBytes)) identity, identityObj, errCode := s.authenticator.AuthenticateRequest(r) switch errCode { case s3err.ErrNone: From ce292a4e5ec2da8328ff54c533dddab94041908d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 01:26:16 -0800 Subject: [PATCH 19/31] s3: require Admin to put or delete a bucket's lifecycle configuration (#12327) PUT and DELETE ?lifecycle were authorized as Write, so an identity granted only object writes on a bucket could add expiration rules or remove the ones the bucket administrator set. They now need Admin on the bucket, like the encryption, public access block and ownership controls writes. Identities authorized by IAM policies are unaffected: they are still checked against s3:PutLifecycleConfiguration. Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3api_server.go | 4 +- weed/s3api/s3api_server_routing_test.go | 59 +++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index b6f8c1486a..dfc2c41ab4 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1106,9 +1106,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // GetBucketLifecycleConfiguration bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "") // PutBucketLifecycleConfiguration - bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("lifecycle", "") // DeleteBucketLifecycleConfiguration - bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_ADMIN)), "DELETE")).Queries("lifecycle", "") // GetBucketLocation bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "") diff --git a/weed/s3api/s3api_server_routing_test.go b/weed/s3api/s3api_server_routing_test.go index 9316174924..b008b18d99 100644 --- a/weed/s3api/s3api_server_routing_test.go +++ b/weed/s3api/s3api_server_routing_test.go @@ -5,6 +5,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "strings" "testing" "time" @@ -529,3 +531,60 @@ func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) { }) } } + +// Write covers a bucket's objects, not its configuration: an identity holding +// Write on the bucket is refused lifecycle writes, while Admin on the bucket or +// a policy granting s3:PutLifecycleConfiguration passes. The bucket is +// negatively cached, so a request that gets past authorization ends at the +// handler's NoSuchBucket. +func TestRouting_BucketLifecycleWritesNeedAdmin(t *testing.T) { + const bucket = "config-bucket" + configFile := filepath.Join(t.TempDir(), "s3.json") + require.NoError(t, os.WriteFile(configFile, []byte(`{ + "identities": [ + {"name":"writer","credentials":[{"accessKey":"WRITERKEY","secretKey":"writersecret"}],"actions":["Read:`+bucket+`","Write:`+bucket+`","List:`+bucket+`","Tagging:`+bucket+`"]}, + {"name":"bucketadmin","credentials":[{"accessKey":"BUCKETADMINKEY","secretKey":"bucketadminsecret"}],"actions":["Admin:`+bucket+`"]}, + {"name":"lifecycler","credentials":[{"accessKey":"LIFECYCLERKEY","secretKey":"lifecyclersecret"}],"policyNames":["LifecyclePolicy"]} + ], + "policies": [ + {"name":"LifecyclePolicy","content":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutLifecycleConfiguration\"],\"Resource\":[\"arn:aws:s3:::`+bucket+`\"]}]}"} + ] +}`), 0600)) + + s3a := setupRoutingTestServer(t) + s3a.iam = NewIdentityAccessManagementWithStore(&S3ApiServerOption{Config: configFile}, nil, "memory") + require.True(t, s3a.iam.isEnabled()) + s3a.cb = &CircuitBreaker{s3a: s3a} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.SetNegativeCache(bucket) + router := mux.NewRouter() + s3a.registerRouter(router) + + send := func(accessKey, secretKey, method, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, "http://localhost:8333/"+bucket+"?lifecycle", strings.NewReader(body)) + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + _, err := v4.NewSigner(creds).Sign(req, strings.NewReader(body), "s3", "us-east-1", time.Now()) + require.NoError(t, err) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr + } + + const lifecycleXML = `rEnabledp/1` + for _, tc := range []struct { + method, body string + }{ + {http.MethodPut, lifecycleXML}, + {http.MethodDelete, ""}, + } { + rr := send("WRITERKEY", "writersecret", tc.method, tc.body) + assert.Equal(t, http.StatusForbidden, rr.Code, "writer %s", tc.method) + assert.Contains(t, rr.Body.String(), "AccessDenied", "writer %s", tc.method) + + for _, key := range [][2]string{{"BUCKETADMINKEY", "bucketadminsecret"}, {"LIFECYCLERKEY", "lifecyclersecret"}} { + rr = send(key[0], key[1], tc.method, tc.body) + assert.Equal(t, http.StatusNotFound, rr.Code, "%s %s", key[0], tc.method) + assert.Contains(t, rr.Body.String(), "NoSuchBucket", "%s %s", key[0], tc.method) + } + } +} From 4850def831f14365461b77ac942b05a387f10b97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20L=E1=BB=99c=20Ph=C3=BAc?= <112325531+omninguyen@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:50:45 +0700 Subject: [PATCH 20/31] fix(s3tables): bind table and view writes to the record they were authorized against (#12328) * s3tables: bind table and view writes to the record they were authorized against * s3tables: bind UpdateNamespace to the namespace it was authorized against UpdateNamespace writes back the whole record it read, owner and creation time included. A namespace deleted and created again before the write took the first one's owner and CreatedAt, which also undoes the CreatedAt binding creates now rely on. Refuse the write with 409 when the namespace at the path has another CreatedAt. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3tables/filer_ops.go | 31 + .../s3tables/handler_bind_authorized_test.go | 585 ++++++++++++++++++ .../handler_delete_create_race_test.go | 19 + weed/s3api/s3tables/handler_maintenance.go | 35 +- weed/s3api/s3tables/handler_namespace.go | 5 + weed/s3api/s3tables/handler_policy.go | 49 +- weed/s3api/s3tables/handler_table.go | 54 +- .../s3tables/handler_update_conflict_test.go | 20 + weed/s3api/s3tables/handler_view.go | 50 +- 9 files changed, 796 insertions(+), 52 deletions(-) create mode 100644 weed/s3api/s3tables/handler_bind_authorized_test.go diff --git a/weed/s3api/s3tables/filer_ops.go b/weed/s3api/s3tables/filer_ops.go index 774743dd69..3775621a6a 100644 --- a/weed/s3api/s3tables/filer_ops.go +++ b/weed/s3api/s3tables/filer_ops.go @@ -643,6 +643,23 @@ func isTableBucketCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool { } } +// isNamespaceCreatedAt is isTableBucketCreatedAt for a namespace its create has +// confirmed: one deleted and created again under the same name reads as the +// authorized one gone. A bucket created again holds only namespaces of its own, +// so a create bound to its namespace this way is bound to the bucket too. +func isNamespaceCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool { + return func(entry *filer_pb.Entry) bool { + if !isFinishedNamespaceEntry(entry) { + return false + } + var metadata namespaceMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil { + return false + } + return metadata.CreatedAt.Equal(createdAt) + } +} + // isNamespaceEntry reports whether entry carries a namespace's catalog record. func isNamespaceEntry(entry *filer_pb.Entry) bool { _, ok := entry.Extended[ExtendedKeyMetadata] @@ -741,6 +758,20 @@ func (h *S3TablesHandler) mutateCatalogRecord( }) } +// requireTableCreatedAt refuses a write to the record at path unless it is the +// table created at createdAt, the one the caller was authorized against: a +// table deleted and created again under the name has a CreatedAt of its own. +// Only the creation time is compared, so the commits an Iceberg writer lands +// meanwhile do not turn a write of the table's policy, tags or maintenance +// settings into a conflict. +func requireTableCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error { + var current tableMetadataInternal + if json.Unmarshal(extended[ExtendedKeyMetadata], ¤t) != nil || !current.CreatedAt.Equal(createdAt) { + return fmt.Errorf("%w: %s holds another table", ErrConcurrentUpdate, path) + } + return nil +} + // confirmParent re-checks the parent after a create wrote written onto // childPath. If the parent is open, it clears the record's unfinished mark — // then checks the parent once more, since a delete claiming it in between diff --git a/weed/s3api/s3tables/handler_bind_authorized_test.go b/weed/s3api/s3tables/handler_bind_authorized_test.go new file mode 100644 index 0000000000..ce48bdf3e1 --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_authorized_test.go @@ -0,0 +1,585 @@ +package s3tables + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// recreatedNamespace is the record of a namespace deleted and created again +// under name: another owner, and a creation time of its own. +func recreatedNamespace(t *testing.T, name string) map[string][]byte { + t.Helper() + meta, err := json.Marshal(namespaceMetadata{ + Namespace: []string{name}, + OwnerAccountID: "111122223333", + CreatedAt: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + return map[string][]byte{ExtendedKeyMetadata: meta} +} + +// replaceNamespaceAtLookup replaces namespace with a recreated one at the first +// lookup of namespace/child, the create's look for its own name, which comes +// after the create was authorized against the first namespace. +func replaceNamespaceAtLookup(t *testing.T, fs *s3tablestest.MemFiler, namespace, child string) map[string][]byte { + t.Helper() + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, namespace) + fs.OnLookup = func(dir, name string) { + if dir != GetNamespacePath(renameTestBucket, namespace) || name != child { + return + } + fs.OnLookup = nil + fs.Put(bucketPath, namespace, recreated) + } + return recreated +} + +// A create is authorized against the namespace it read. If that namespace is +// deleted and another is created under the same name before the create writes, +// the entry would land in a namespace nobody authorized it for, carrying the +// first one's owner. The create checks the namespace by its creation time, +// reads the new one as the old one gone, and backs out what it wrote. +func TestCreateInNamespaceTreatsARecreatedNamespaceAsGone(t *testing.T) { + for _, tc := range []struct { + operation string + input func(t *testing.T) interface{} + }{ + {"CreateTable", func(t *testing.T) interface{} { return createTableRequest(t, "ns", "t") }}, + {"RegisterTable", func(t *testing.T) interface{} { + return &RegisterTableRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v1.metadata.json", + } + }}, + {"CreateView", func(t *testing.T) interface{} { return createViewRequest(t, "t") }}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + recreated := replaceNamespaceAtLookup(t, fs, "ns", "t") + + code, resp := runHandler(t, m, fs.Client, tc.operation, tc.input(t)) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "nothing is left in the new namespace") + require.NotNil(t, fs.Get(bucketPath, "ns"), "the new namespace stays") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "and is left as it was created") + }) + } +} + +// A rename is authorized to create in the destination namespace it read. If +// that namespace is deleted and created again before the rename publishes, the +// renamed table would land in a namespace nobody authorized it for. The rename +// reads the new one as the old one gone, backs out, and keeps the source. +func TestRenameTableTreatsARecreatedDestinationAsGone(t *testing.T) { + fs, m := startRenameManager(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := GetNamespacePath(renameTestBucket, "ns") + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + source := fs.Get(nsPath, "t").Extended + // The rename looks the destination name up right after it read the + // destination namespace. + recreated := replaceNamespaceAtLookup(t, fs, "ns2", "t") + + code, resp := runHandler(t, m, fs.Client, "RenameTable", &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: "t", + DestNamespace: []string{"ns2"}, + DestName: "t", + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns2"), "t"), "nothing is left in the new namespace") + require.NotNil(t, fs.Get(bucketPath, "ns2"), "the new namespace stays") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns2").Extended, "and is left as it was created") + require.NotNil(t, fs.Get(nsPath, "t"), "the source stays") + assert.Equal(t, source, fs.Get(nsPath, "t").Extended, "with its record and no claim") +} + +// recreatedRecord is the record of a table or view deleted and created again +// under ns/name: another owner, and a creation time of its own. +func recreatedRecord(t *testing.T, name, entryType string) map[string][]byte { + t.Helper() + now := time.Now().Add(time.Hour) + meta, err := json.Marshal(tableMetadataInternal{ + Name: name, + Namespace: "ns", + Format: FormatIceberg, + CreatedAt: now, + ModifiedAt: now, + OwnerAccountID: "111122223333", + VersionToken: generateVersionToken(), + MetadataVersion: 1, + MetadataLocation: "s3://" + renameTestBucket + "/ns/" + name + "/metadata/v1-recreated.metadata.json", + }) + require.NoError(t, err) + return map[string][]byte{ + ExtendedKeyMetadata: meta, + ExtendedKeyEntryType: []byte(entryType), + } +} + +// startViewV seeds the namespace ns holding the view v. +func startViewV(t *testing.T) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + requireOK(t, m, fs, "CreateView", createViewRequest(t, "v")) + return fs, m +} + +func updateViewRequest(t *testing.T, token, version string) *UpdateViewRequest { + return &UpdateViewRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "v", + VersionToken: token, + MetadataLocation: "s3://" + renameTestBucket + "/ns/v/metadata/" + version + ".metadata.json", + } +} + +// An update is authorized against the view it read. If that view is deleted +// and another is created under the same name before the update writes, the +// write would land on a view nobody authorized it for. The write is conditional +// on the record the update read, so it is refused and the new view kept. +func TestUpdateViewSparesARecreatedView(t *testing.T) { + fs, m := startViewV(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + recreated := recreatedRecord(t, "v", EntryTypeView) + // The update's write is its first; the view is replaced as it lands, so the + // write loses and its retry reads the new view. + fs.BeforeUpdate = func() { fs.Put(nsPath, "v", recreated) } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, "", "v2")) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(nsPath, "v"), "the new view stays") + assert.Equal(t, recreated, fs.Get(nsPath, "v").Extended, "and is left as it was created") +} + +// Two updates that read the same version both pass the version-token check. +// The second to write must not overwrite the first one's metadata pointer, or +// the version the first published is lost. +func TestUpdateViewRejectsLostUpdate(t *testing.T) { + fs, m := startViewV(t) + token := versionTokenOf(t, fs, "v") + var winnerStatus int + var winnerResp S3TablesError + // The winner runs whole between the loser's read and its write. + fs.BeforeUpdate = func() { + winnerStatus, winnerResp = runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, token, "v2-winner")) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, token, "v2-loser")) + + require.Equal(t, http.StatusOK, winnerStatus, "%+v", winnerResp) + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + got := &GetViewResponse{} + require.NoError(t, m.Execute(context.Background(), NewManagerClient(fs.Client), "GetView", getViewRequest(t, "v"), got, "")) + assert.Equal(t, updateViewRequest(t, token, "v2-winner").MetadataLocation, got.MetadataLocation, "the first update must survive") +} + +// A policy written between the authorization check and the write decides +// whether this caller may still update the view, so the update is refused +// rather than applied on a decision that has been overtaken. +func TestUpdateViewRejectsWhenTheAuthorizingPolicyChanges(t *testing.T) { + fs, m := startViewV(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + policy := []byte(`{"Version":"2012-10-17","Statement":[]}`) + fs.BeforeUpdate = func() { + extended := map[string][]byte{} + for key, value := range fs.Get(nsPath, "v").Extended { + extended[key] = value + } + extended[ExtendedKeyPolicy] = policy + fs.Put(nsPath, "v", extended) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, "", "v2")) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Equal(t, policy, fs.Get(nsPath, "v").Extended[ExtendedKeyPolicy], "the concurrent policy write must survive") +} + +const bindTestPolicy = `{"Statement":[{"Effect":"Allow","Principal":"writer","Action":"s3tables:*","Resource":"*"}]}` + +// tableWriteOp is a write to the table ns/t, with the attributes the table +// carries for it besides its record. +type tableWriteOp struct { + operation string + input func(t *testing.T) interface{} + extra map[string][]byte +} + +func tableWriteOps() []tableWriteOp { + tableARN := func(t *testing.T) string { + arn, err := BuildTableARN(DefaultRegion, DefaultAccountID, renameTestBucket, "ns", "t") + require.NoError(t, err) + return arn + } + return []tableWriteOp{ + {operation: "PutTablePolicy", input: func(t *testing.T) interface{} { + return &PutTablePolicyRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}, Name: "t", ResourcePolicy: bindTestPolicy} + }}, + {operation: "DeleteTablePolicy", input: func(t *testing.T) interface{} { + return &DeleteTablePolicyRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}, Name: "t"} + }, extra: map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}}, + {operation: "TagResource", input: func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: tableARN(t), Tags: map[string]string{"k": "v"}} + }}, + {operation: "UntagResource", input: func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: tableARN(t), TagKeys: []string{"k"}} + }, extra: map[string][]byte{ExtendedKeyTags: []byte(`{"k":"v","keep":"1"}`)}}, + {operation: "PutTableMaintenanceConfiguration", input: func(t *testing.T) interface{} { + return &PutTableMaintenanceConfigurationRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + Type: MaintenanceTypeIcebergCompaction, + Value: &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}, + } + }}, + } +} + +// startTableT seeds the namespace ns holding the table t, carrying extra +// besides its record. +func startTableT(t *testing.T, extra map[string][]byte) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs, m := startRenameManager(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + extended := extendedOf(t, fs, nsPath, "t") + for key, value := range extra { + extended[key] = value + } + fs.Put(nsPath, "t", extended) + return fs, m +} + +// A write to a table's policy, tags or maintenance settings is authorized +// against the table it read. If that table is deleted and another is created +// under the same name before the write, it would land on a table nobody +// authorized it for: the policy or tag values it compares can match the new +// table's just as well. The write checks the table by its creation time and is +// refused, and the new table is kept as it was created. +func TestTableWritesSpareARecreatedTable(t *testing.T) { + for _, op := range tableWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startTableT(t, op.extra) + nsPath := GetNamespacePath(renameTestBucket, "ns") + recreated := recreatedRecord(t, "t", EntryTypeTable) + for key, value := range op.extra { + recreated[key] = value + } + // The write is the operation's first; the table is replaced as it + // lands, so the write loses and its retry reads the new table. + fs.BeforeUpdate = func() { fs.Put(nsPath, "t", recreated) } + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(nsPath, "t"), "the new table stays") + assert.Equal(t, recreated, fs.Get(nsPath, "t").Extended, "and is left as it was created") + }) + } +} + +// The same writes compare only the table's creation time, not its metadata: +// an Iceberg commit to the table between the read and the write is the same +// table, and must not turn the write into a conflict. +func TestTableWritesTolerateACommitInBetween(t *testing.T) { + for _, op := range tableWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startTableT(t, op.extra) + nsPath := GetNamespacePath(renameTestBucket, "ns") + committedLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json" + fs.BeforeUpdate = func() { + extended := extendedOf(t, fs, nsPath, "t") + var metadata tableMetadataInternal + require.NoError(t, json.Unmarshal(extended[ExtendedKeyMetadata], &metadata)) + metadata.MetadataVersion = 4 + metadata.MetadataLocation = committedLocation + metadata.VersionToken = generateVersionToken() + metadata.ModifiedAt = time.Now() + data, err := json.Marshal(metadata) + require.NoError(t, err) + extended[ExtendedKeyMetadata] = data + fs.Put(nsPath, "t", extended) + } + + requireOK(t, m, fs, op.operation, op.input(t)) + + got, err := runGetTable(t, m, fs, "ns", "t") + require.NoError(t, err) + assert.Equal(t, committedLocation, got.MetadataLocation, "the commit stays") + }) + } +} + +// namespaceMetadataOf reads the record of a namespace in the test bucket. +func namespaceMetadataOf(t *testing.T, fs *s3tablestest.MemFiler, namespace string) namespaceMetadata { + t.Helper() + entry := fs.Get(GetTableBucketPath(renameTestBucket), namespace) + require.NotNil(t, entry) + var metadata namespaceMetadata + require.NoError(t, json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata)) + return metadata +} + +// updateNamespaceAtLookup runs a real UpdateNamespace the first time child is +// looked up under namespace: after the request was authorized against the +// namespace and before it writes. The answer fills in once the update has run. +func updateNamespaceAtLookup(t *testing.T, fs *s3tablestest.MemFiler, m *Manager, namespace, child string) *handlerResult { + t.Helper() + update := &handlerResult{} + fs.OnLookup = func(dir, name string) { + if dir != GetNamespacePath(renameTestBucket, namespace) || name != child { + return + } + fs.OnLookup = nil + update.status, update.resp = runHandler(t, m, fs.Client, "UpdateNamespace", &UpdateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{namespace}, + Properties: map[string]string{"edited": "true"}, + }) + } + return update +} + +// requireNamespaceUpdatedInPlace checks that the update ran, took effect, and +// kept the namespace the record it was: same owner, same creation time. +func requireNamespaceUpdatedInPlace(t *testing.T, fs *s3tablestest.MemFiler, namespace string, before namespaceMetadata, update *handlerResult) { + t.Helper() + require.NotZero(t, update.status, "the update has to land between the authorization and the write") + require.Equal(t, http.StatusOK, update.status, "%+v", update.resp) + after := namespaceMetadataOf(t, fs, namespace) + assert.Equal(t, map[string]string{"edited": "true"}, after.Properties) + assert.True(t, after.CreatedAt.Equal(before.CreatedAt), "an update must keep the creation time: %v became %v", before.CreatedAt, after.CreatedAt) + assert.Equal(t, before.OwnerAccountID, after.OwnerAccountID) +} + +// A namespace update edits the record a request was authorized against; it +// does not replace it. The creation time survives, so the request still writes: +// reading the edit as a delete and create would back the entry out and answer +// 404 for a namespace that never went away. Unlike +// TestCreateConfirmingAcrossANamespaceUpdateSucceeds, the update here is the +// real UpdateNamespace and lands before the first check against the authorized +// record, not after it. +func TestCreateAuthorizedBeforeANamespaceUpdateSucceeds(t *testing.T) { + for _, op := range createOps(t) { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + before := namespaceMetadataOf(t, fs, "ns") + update := updateNamespaceAtLookup(t, fs, m, "ns", lateEntry) + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + requireNamespaceUpdatedInPlace(t, fs, "ns", before, update) + entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), lateEntry) + require.NotNil(t, entry, "the entry the request answered for must exist") + assert.Contains(t, entry.Extended, ExtendedKeyMetadata) + assert.NotContains(t, entry.Extended, ExtendedKeyDeleting, "and carry no unfinished mark") + }) + } + + // A rename is bound to its destination namespace the same way. + for _, tc := range []struct { + operation string + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, name string) + }{ + {"RenameTable", "t", nil}, + {"RenameView", "v", seedView}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startRenameManager(t) + if tc.seed != nil { + tc.seed(t, fs, tc.name) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + before := namespaceMetadataOf(t, fs, "ns2") + update := updateNamespaceAtLookup(t, fs, m, "ns2", tc.name) + + code, resp := runHandler(t, m, fs.Client, tc.operation, &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: tc.name, + DestNamespace: []string{"ns2"}, + DestName: tc.name, + }) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + requireNamespaceUpdatedInPlace(t, fs, "ns2", before, update) + dest := fs.Get(GetNamespacePath(renameTestBucket, "ns2"), tc.name) + require.NotNil(t, dest, "the renamed entry must exist") + assert.Contains(t, dest.Extended, ExtendedKeyMetadata) + assert.NotContains(t, dest.Extended, ExtendedKeyDeleting, "and carry no unfinished mark") + }) + } +} + +// writeRecorder remembers the writes that reach a directory tree, so a test can +// tell a request that never wrote there from one that wrote and backed out. +type writeRecorder struct { + filer_pb.SeaweedFilerClient + tree string + mu sync.Mutex + writes []string +} + +func (c *writeRecorder) note(op, dir, name string) { + if dir != c.tree && !strings.HasPrefix(dir, c.tree+"/") { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.writes = append(c.writes, op+" "+dir+"/"+name) +} + +func (c *writeRecorder) CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) { + c.note("create", in.Directory, in.Entry.GetName()) + return c.SeaweedFilerClient.CreateEntry(ctx, in, opts...) +} + +func (c *writeRecorder) UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) { + c.note("update", in.Directory, in.Entry.GetName()) + return c.SeaweedFilerClient.UpdateEntry(ctx, in, opts...) +} + +func (c *writeRecorder) DeleteEntry(ctx context.Context, in *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) { + c.note("delete", in.Directory, in.Name) + return c.SeaweedFilerClient.DeleteEntry(ctx, in, opts...) +} + +// A namespace deleted and created again between the authorization and the +// early check is not the one the caller was authorized against, so the request +// is refused before it writes anything: the check after the write only backs +// the record out, and a back-out the filer turns away would leave it in a +// namespace nobody authorized it for. +func TestCreateRefusedBeforeWritingIntoAReplacedNamespace(t *testing.T) { + for _, op := range createOps(t) { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + // The create reads the namespace record three times before it writes: for + // its metadata, for its policy, and for the early check. The namespace is + // replaced as the third read starts. + const earlyCheck = 3 + reads := 0 + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + if reads++; reads == earlyCheck { + fs.OnLookup = nil + fs.Put(bucketPath, "ns", recreated) + } + } + recorder := &writeRecorder{SeaweedFilerClient: fs.Client, tree: GetNamespacePath(renameTestBucket, "ns")} + + code, resp := runHandler(t, m, recorder, op.operation, op.input) + + require.GreaterOrEqual(t, reads, earlyCheck, "the namespace has to be replaced before the early check") + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Empty(t, recorder.writes, "nothing is written into the namespace that replaced the authorized one") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "and it is left as it was created") + }) + } + + // A rename reads its destination namespace and then the destination name, + // both before the early check. + for _, tc := range []struct { + operation string + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, name string) + }{ + {"RenameTable", "t", nil}, + {"RenameView", "v", seedView}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startRenameManager(t) + if tc.seed != nil { + tc.seed(t, fs, tc.name) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + recreated := replaceNamespaceAtLookup(t, fs, "ns2", tc.name) + recorder := &writeRecorder{SeaweedFilerClient: fs.Client, tree: GetNamespacePath(renameTestBucket, "ns2")} + + code, resp := runHandler(t, m, recorder, tc.operation, &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: tc.name, + DestNamespace: []string{"ns2"}, + DestName: tc.name, + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Empty(t, recorder.writes, "nothing is written into the namespace that replaced the authorized one") + assert.Equal(t, recreated, fs.Get(GetTableBucketPath(renameTestBucket), "ns2").Extended, "and it is left as it was created") + }) + } +} + +// UpdateNamespace writes the record it was authorized against, owner and +// creation time included. A namespace deleted and created again before that +// write is refused, not handed the first one's owner. +func TestUpdateNamespaceSparesARecreatedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnUpdate = nil + fs.Put(bucketPath, "ns", recreated) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateNamespace", &UpdateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Properties: map[string]string{"edited": "true"}, + }) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "the new namespace is untouched") +} diff --git a/weed/s3api/s3tables/handler_delete_create_race_test.go b/weed/s3api/s3tables/handler_delete_create_race_test.go index b46b9fb8d0..7f51145d10 100644 --- a/weed/s3api/s3tables/handler_delete_create_race_test.go +++ b/weed/s3api/s3tables/handler_delete_create_race_test.go @@ -512,6 +512,25 @@ func TestUpdateRefusedOnAMarkedRecord(t *testing.T) { assert.Equal(t, ErrCodeNoSuchTable, resp.Type) } +// DeleteTablePolicy refuses a marked record the same way: the policy it would +// remove sits on an entry every reader reports absent. +func TestDeleteTablePolicyRefusedOnAMarkedRecord(t *testing.T) { + fs, m := startTableT(t, map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}) + nsPath := GetNamespacePath(renameTestBucket, "ns") + fs.BeforeUpdate = func() { plantCreateMark(fs, nsPath, "t") } + + status, resp := runHandler(t, m, fs.Client, "DeleteTablePolicy", &DeleteTablePolicyRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + }) + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchTable, resp.Type) + extended := fs.Get(nsPath, "t").Extended + assert.Equal(t, []byte(bindTestPolicy), extended[ExtendedKeyPolicy], "the refused delete keeps the policy") + assert.Equal(t, []byte(createClaim), extended[ExtendedKeyDeleting], "and leaves the create's mark for its back-out") +} + // A rename into a namespace a delete claimed is refused before it touches the // source: the destination cannot be confirmed, so the name must stay where it // was. diff --git a/weed/s3api/s3tables/handler_maintenance.go b/weed/s3api/s3tables/handler_maintenance.go index 02a2764a25..f359f0a38b 100644 --- a/weed/s3api/s3tables/handler_maintenance.go +++ b/weed/s3api/s3tables/handler_maintenance.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) @@ -50,7 +51,7 @@ func (h *S3TablesHandler) handlePutTableBucketMaintenanceConfiguration(w http.Re return err } - if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value); err != nil { + if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value, nil); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err } @@ -116,13 +117,17 @@ func (h *S3TablesHandler) handlePutTableMaintenanceConfiguration(w http.Response return err } - if _, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName); err != nil { + _, createdAt, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName) + if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err } tablePath := GetTablePath(bucketName, namespaceName, tableName) - if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value); err != nil { + sameTable := func(extended map[string][]byte) error { + return requireTableCreatedAt(extended, tablePath, createdAt) + } + if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value, sameTable); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err } @@ -145,7 +150,7 @@ func (h *S3TablesHandler) handleGetTableMaintenanceConfiguration(w http.Response return err } - tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName) + tableARN, _, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err @@ -181,7 +186,7 @@ func (h *S3TablesHandler) handleGetTableMaintenanceJobStatus(w http.ResponseWrit return err } - tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName) + tableARN, _, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err @@ -253,18 +258,25 @@ func buildJobStatusResponse(recorded MaintenanceJobStatus, config MaintenanceCon // putMaintenanceConfiguration merges one maintenance type into the stored // configuration, leaving the other types alone. The write checks the catalog // identity in the same conditional mutation, so it cannot land on a name a -// concurrent rename or delete has already soft-deleted. +// concurrent rename or delete has already soft-deleted. sameRecord, when set, +// refuses a record other than the one the caller was authorized against. func (h *S3TablesHandler) putMaintenanceConfiguration( r *http.Request, filerClient FilerClient, resourcePath, maintenanceType string, value *MaintenanceConfigurationValue, + sameRecord func(extended map[string][]byte) error, ) error { return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { if len(extended[ExtendedKeyMetadata]) == 0 { return fmt.Errorf("%w: %s", filer_pb.ErrNotFound, resourcePath) } + if sameRecord != nil { + if err := sameRecord(extended); err != nil { + return err + } + } config := MaintenanceConfiguration{} if current := extended[ExtendedKeyMaintenance]; len(current) > 0 { @@ -362,8 +374,9 @@ func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClien } // authorizeMaintenanceTable checks the caller may perform operation on the -// table, returning the table ARN. -func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, error) { +// table, returning the table ARN and the creation time of the table record the +// check read. +func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, time.Time, error) { tablePath := GetTablePath(bucketName, namespaceName, tableName) bucketPath := GetTableBucketPath(bucketName) @@ -391,7 +404,7 @@ func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient return err }) if err != nil { - return "", err + return "", time.Time{}, err } tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) @@ -409,10 +422,10 @@ func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient } if deniedByPolicy(operation, principal, tablePolicy, tableARN, ctx) || !CheckPermissionWithContext(operation, principal, metadata.OwnerAccountID, bucketPolicy, tableARN, ctx) { - return "", NewAuthError(operation, principal, "not authorized to "+operation) + return "", time.Time{}, NewAuthError(operation, principal, "not authorized to "+operation) } - return tableARN, nil + return tableARN, metadata.CreatedAt, nil } // readPolicy returns the policy stored on path, or an empty string when the diff --git a/weed/s3api/s3tables/handler_namespace.go b/weed/s3api/s3tables/handler_namespace.go index 2683e06fd0..ccd00298ba 100644 --- a/weed/s3api/s3tables/handler_namespace.go +++ b/weed/s3api/s3tables/handler_namespace.go @@ -495,6 +495,11 @@ func (h *S3TablesHandler) handleUpdateNamespace(w http.ResponseWriter, r *http.R err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, namespacePath, func(extended map[string][]byte) error { + // The write carries the authorized record's owner and creation + // time, so it must not land on a namespace created again since. + if !isNamespaceCreatedAt(metadata.CreatedAt)(&filer_pb.Entry{Extended: extended}) { + return fmt.Errorf("%w: %s holds another namespace", ErrConcurrentUpdate, namespacePath) + } extended[ExtendedKeyMetadata] = metadataBytes return nil }) diff --git a/weed/s3api/s3tables/handler_policy.go b/weed/s3api/s3tables/handler_policy.go index f6c89f6061..25cf2fcd12 100644 --- a/weed/s3api/s3tables/handler_policy.go +++ b/weed/s3api/s3tables/handler_policy.go @@ -7,10 +7,21 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) +// tableCreatedAtOf returns the creation time a table's metadata records, which +// tells the table apart from one created again under its name. +func tableCreatedAtOf(data []byte) (time.Time, error) { + var meta tableMetadataInternal + if err := json.Unmarshal(data, &meta); err != nil { + return time.Time{}, err + } + return meta.CreatedAt, nil +} + // extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata. // This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource, // and handleUntagResource. @@ -401,6 +412,9 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, tablePath, func(extended map[string][]byte) error { + if err := requireTableCreatedAt(extended, tablePath, metadata.CreatedAt); err != nil { + return err + } if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(tablePolicy)) { return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) } @@ -617,15 +631,26 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy") } + // The removal is conditional on the table and the policy this request was + // authorized against, as PutTablePolicy's write is. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.removeExtendedAttributesIf(r.Context(), client, tablePath, - map[string][]byte{ExtendedKeyPolicy: []byte(tablePolicy)}, - ExtendedKeyPolicy) + return h.mutateCatalogRecord(r.Context(), client, tablePath, func(extended map[string][]byte) error { + if err := requireTableCreatedAt(extended, tablePath, metadata.CreatedAt); err != nil { + return err + } + if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(tablePolicy)) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } + delete(extended, ExtendedKeyPolicy) + return nil + }) }) if err != nil && !errors.Is(err, ErrAttributeNotFound) { if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "table policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table policy") } @@ -666,6 +691,7 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string + var tableCreatedAt time.Time requestTagKeys := mapKeys(req.Tags) err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check @@ -695,6 +721,9 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque } if rType == ResourceTypeTable { + if tableCreatedAt, err = tableCreatedAtOf(data); err != nil { + return err + } if _, tableNamespace, tableName, err = parseTableFromARN(req.ResourceARN); err != nil { return err } @@ -769,6 +798,11 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { + if rType == ResourceTypeTable { + if err := requireTableCreatedAt(extended, resourcePath, tableCreatedAt); err != nil { + return err + } + } extended[extendedKey] = tagsBytes return nil }) @@ -942,6 +976,7 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string + var tableCreatedAt time.Time err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata) @@ -970,6 +1005,9 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req } if rType == ResourceTypeTable { + if tableCreatedAt, err = tableCreatedAtOf(data); err != nil { + return err + } if _, tableNamespace, tableName, err = parseTableFromARN(req.ResourceARN); err != nil { return err } @@ -1042,6 +1080,11 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { + if rType == ResourceTypeTable { + if err := requireTableCreatedAt(extended, resourcePath, tableCreatedAt); err != nil { + return err + } + } extended[extendedKey] = tagsBytes return nil }) diff --git a/weed/s3api/s3tables/handler_table.go b/weed/s3api/s3tables/handler_table.go index 10edb2e94d..afa5a2ddc9 100644 --- a/weed/s3api/s3tables/handler_table.go +++ b/weed/s3api/s3tables/handler_table.go @@ -166,7 +166,7 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque if err := h.claimCatalogEntry(r.Context(), client, tablePath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceEntry, tablePath, attrs, dataPath, tablePath) + return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceCreatedAt(target.namespaceCreatedAt), tablePath, attrs, dataPath, tablePath) }) if err != nil { @@ -313,7 +313,7 @@ func (h *S3TablesHandler) handleRegisterTable(w http.ResponseWriter, r *http.Req if err := h.claimCatalogEntry(r.Context(), client, tablePath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceEntry, tablePath, attrs, tablePath) + return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceCreatedAt(target.namespaceCreatedAt), tablePath, attrs, tablePath) }) if err != nil { if errors.Is(err, ErrNameTaken) { @@ -1636,7 +1636,10 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ h.writeError(w, http.StatusConflict, kind.existsCode, fmt.Sprintf("%s %s already exists", kind.noun, destName)) return ErrTableAlreadyExists } - if err := h.refuseClaimedNamespace(w, r, filerClient, destNamespacePath, destNamespace); err != nil { + // The destination is bound to the namespace the create check ran against: + // one deleted and created again under its name reads as gone. + isDestNamespace := isNamespaceCreatedAt(destNamespaceMetadata.CreatedAt) + if err := h.refuseClaimedNamespace(w, r, filerClient, destNamespacePath, destNamespace, isDestNamespace); err != nil { return err } @@ -1718,7 +1721,7 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ h.removeLeftoverDirectory(ctx, client, destPath) return releaseClaim(err) } - destParent, err := h.openParent(r.Context(), client, destNamespacePath, isNamespaceEntry) + destParent, err := h.openParent(r.Context(), client, destNamespacePath, isDestNamespace) if err != nil { return backOutDest(err) } @@ -1744,7 +1747,7 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ destConfirmed := false if err = h.removeExtendedAttributesIf(commitCtx, client, destPath, written, ExtendedKeyDeleting); err == nil { destConfirmed = true - err = h.parentStillOpen(commitCtx, client, destNamespacePath, isNamespaceEntry, destParent) + err = h.parentStillOpen(commitCtx, client, destNamespacePath, isDestNamespace, destParent) } if err != nil { if destConfirmed { @@ -2116,13 +2119,15 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque } // createTableTarget is the namespace a create or register resolved to, once the -// caller has been authorized to put a table there. +// caller has been authorized to put a table there. namespaceCreatedAt names the +// namespace record the authorization read, so the write lands in no other. type createTableTarget struct { - bucketName string - namespaceName string - tableName string - bucketFormat string - ownerAccountID string + bucketName string + namespaceName string + tableName string + bucketFormat string + ownerAccountID string + namespaceCreatedAt time.Time } // authorizeCreateTable validates the names a create names and checks the caller @@ -2256,17 +2261,18 @@ func (h *S3TablesHandler) authorizeCreateTable(w http.ResponseWriter, r *http.Re } if namespaceExists { - if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName); err != nil { + if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName, isNamespaceCreatedAt(nsMetadata.CreatedAt)); err != nil { return nil, err } } return &createTableTarget{ - bucketName: bucketName, - namespaceName: namespaceName, - tableName: tableName, - bucketFormat: bucketMetadata.Format, - ownerAccountID: nsMetadata.OwnerAccountID, + bucketName: bucketName, + namespaceName: namespaceName, + tableName: tableName, + bucketFormat: bucketMetadata.Format, + ownerAccountID: nsMetadata.OwnerAccountID, + namespaceCreatedAt: nsMetadata.CreatedAt, }, nil } @@ -2289,14 +2295,16 @@ func (h *S3TablesHandler) writeNamespaceClosedError(w http.ResponseWriter, err e return true } -// refuseClaimedNamespace answers 409 when a delete has claimed the namespace or -// its create is not confirmed yet. Call it after the authorization check, so -// denied callers never see the delete. -func (h *S3TablesHandler) refuseClaimedNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient, namespacePath, namespaceName string) error { +// refuseClaimedNamespace refuses a request before it writes anything when the +// namespace is not the open record the caller was authorized against: 409 when a +// delete has claimed it, 404 when it is gone, was replaced by another namespace, +// or its create is not confirmed yet. isNamespace accepts that record. Call it +// after the authorization check, so denied callers never see the delete. +func (h *S3TablesHandler) refuseClaimedNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient, namespacePath, namespaceName string, isNamespace func(*filer_pb.Entry) bool) error { err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.checkParentOpen(r.Context(), client, namespacePath, isNamespaceEntry) + return h.checkParentOpen(r.Context(), client, namespacePath, isNamespace) }) - if errors.Is(err, errParentDeleting) || errors.Is(err, errParentCreating) { + if errors.Is(err, errParentDeleting) || errors.Is(err, errParentCreating) || errors.Is(err, errParentGone) { h.writeNamespaceClosedError(w, err, namespaceName) return err } diff --git a/weed/s3api/s3tables/handler_update_conflict_test.go b/weed/s3api/s3tables/handler_update_conflict_test.go index 0117f736dc..5b4066b772 100644 --- a/weed/s3api/s3tables/handler_update_conflict_test.go +++ b/weed/s3api/s3tables/handler_update_conflict_test.go @@ -67,3 +67,23 @@ func TestUntagResourceClaimedRecordConflicts(t *testing.T) { assert.Equal(t, http.StatusConflict, status, "%+v", resp) assert.Equal(t, ErrCodeConflict, resp.Type) } + +// DeleteTablePolicy writes like every other mutation of a table record, so a +// record another operation claimed refuses it with 409 instead of losing the +// policy under the claim's owner. +func TestDeleteTablePolicyClaimedRecordConflicts(t *testing.T) { + fs, m := startTableT(t, map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}) + nsPath := GetNamespacePath(renameTestBucket, "ns") + claimRecord(t, fs, nsPath, "t") + + status, resp := runHandler(t, m, fs.Client, "DeleteTablePolicy", &DeleteTablePolicyRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + }) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + extended := fs.Get(nsPath, "t").Extended + assert.Equal(t, []byte(bindTestPolicy), extended[ExtendedKeyPolicy], "the refused delete keeps the policy") + assert.Equal(t, []byte("delete"), extended[ExtendedKeyDeleting], "and leaves the claim to its owner") +} diff --git a/weed/s3api/s3tables/handler_view.go b/weed/s3api/s3tables/handler_view.go index cb63a3b97c..e8f6dadb61 100644 --- a/weed/s3api/s3tables/handler_view.go +++ b/weed/s3api/s3tables/handler_view.go @@ -1,6 +1,7 @@ package s3tables import ( + "bytes" "context" "encoding/json" "errors" @@ -85,7 +86,9 @@ func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Reques h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create view in this namespace") return ErrAccessDenied } - if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName); err != nil { + // The view is bound to the namespace record it was authorized against. + isNamespace := isNamespaceCreatedAt(namespaceMetadata.CreatedAt) + if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName, isNamespace); err != nil { return err } @@ -169,7 +172,7 @@ func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Reques if err := h.claimCatalogEntry(r.Context(), client, viewPath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, namespacePath, isNamespaceEntry, viewPath, attrs, viewPath) + return h.confirmParent(r.Context(), client, namespacePath, isNamespace, viewPath, attrs, viewPath) }) if err != nil { if errors.Is(err, ErrNameTaken) { @@ -432,14 +435,17 @@ func (h *S3TablesHandler) listViewsInNamespace(r *http.Request, client filer_pb. } // updateViewTarget is the view state an update was authorized against: the -// metadata the version-token check reads and the fields the response echoes. +// metadata the version-token check reads, the stored values the +// optimistic-concurrency check compares and the fields the response echoes. type updateViewTarget struct { - bucketName string - namespaceName string - viewName string - viewPath string - viewARN string - metadata tableMetadataInternal + bucketName string + namespaceName string + viewName string + viewPath string + viewARN string + metadata tableMetadataInternal + storedMetadata []byte + storedPolicy []byte } // authorizeUpdateView validates the names an update names, loads the view and @@ -460,6 +466,7 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req viewPath := GetTablePath(bucketName, namespaceName, viewName) var metadata tableMetadataInternal + var storedMetadata []byte err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { entry, err := h.lookupEntry(r.Context(), client, viewPath) if err != nil { @@ -472,6 +479,7 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req if !ok { return fmt.Errorf("%w: %w: %s", filer_pb.ErrNotFound, ErrAttributeNotFound, ExtendedKeyMetadata) } + storedMetadata = data return json.Unmarshal(data, &metadata) }) if err != nil { @@ -496,12 +504,14 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req } return &updateViewTarget{ - bucketName: bucketName, - namespaceName: namespaceName, - viewName: viewName, - viewPath: viewPath, - viewARN: viewARN, - metadata: metadata, + bucketName: bucketName, + namespaceName: namespaceName, + viewName: viewName, + viewPath: viewPath, + viewARN: viewARN, + metadata: metadata, + storedMetadata: storedMetadata, + storedPolicy: []byte(viewPolicy), }, nil } @@ -558,8 +568,18 @@ func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Reques return err } + // Conditional on the record this request read and authorized against, as + // UpdateTable's write is: a concurrent update would otherwise be dropped, + // and a view deleted and created again under the name would take a write + // nobody authorized for it. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, viewPath, func(extended map[string][]byte) error { + if !bytes.Equal(extended[ExtendedKeyMetadata], target.storedMetadata) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyMetadata) + } + if !bytes.Equal(extended[ExtendedKeyPolicy], target.storedPolicy) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } extended[ExtendedKeyMetadata] = metadataBytes return nil }) From 66a6598fddbcf39c7b3ed968f3780222867bd0cc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 03:00:30 -0800 Subject: [PATCH 21/31] fix(s3tables,s3vectors): bind table and vector bucket writes to the bucket they were authorized against (#12329) * s3tables: bind table bucket policy writes to the authorized bucket PutTableBucketPolicy compared only the policy bytes, and DeleteTableBucketPolicy removed the policy unconditionally. A bucket deleted and created again under the same name between authorization and the write took a policy change nobody authorized for it. Both writes now require the bucket's CreatedAt and the policy they were authorized against, answer 409 otherwise and 404 for a bucket that is gone. Co-Authored-By: Claude Opus 5.5 * s3tables: bind bucket tag writes to the authorized bucket TagResource and UntagResource checked the creation time only for a table ARN. On a bucket ARN the merged tags landed on a bucket deleted and created again since authorization. extractResourceOwnerAndBucket now returns the resource's CreatedAt for both kinds, and the write requires it. Co-Authored-By: Claude Opus 5.5 * s3tables: bind bucket maintenance writes to the authorized bucket PutTableBucketMaintenanceConfiguration wrote onto whatever bucket held the name at write time. authorizeMaintenanceBucket now returns the bucket's CreatedAt like authorizeMaintenanceTable, and putMaintenanceConfiguration takes the check and creation time from both callers instead of an optional closure. Co-Authored-By: Claude Opus 5.5 * s3vectors: bind vector bucket policy writes to the authorized bucket Put/DeleteVectorBucketPolicy authorized inside the compare-and-swap with the owner from an earlier bucket read, so a bucket deleted and created again by another account in between had its policy rewritten by the old owner. The mutation now requires the entry's createdAt to match the record it authorized against. Co-Authored-By: Claude Opus 5.5 * s3vectors: bind DeleteVectorBucket to the authorized bucket DeleteVectorBucket authorized one bucket record, then ran the trusted catalog DeleteTableBucket on whatever bucket held the name by then, so an empty bucket deleted and created again by another account in between was deleted by the old owner. The catalog delete now accepts the authorized bucket's createdAt through an in-process context value and treats any other bucket as gone. Co-Authored-By: Claude Opus 5.5 * s3vectors: bind CreateIndex and DeleteIndex to the authorized bucket Both authorized the bucket, then ran trusted CreateNamespace, CreateTable or DeleteTable on whatever bucket held the name by then, so an index could be created in, or deleted from, a bucket another account created again in between. Those catalog operations now honor the bucket binding DeleteTableBucket already takes, and the vector handlers pass the authorized bucket's createdAt. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3tables/filer_ops.go | 16 +- .../s3tables/handler_bind_bucket_test.go | 104 +++++++++ .../handler_bucket_get_list_delete.go | 3 + weed/s3api/s3tables/handler_maintenance.go | 37 ++-- weed/s3api/s3tables/handler_namespace.go | 5 + weed/s3api/s3tables/handler_policy.go | 93 ++++---- weed/s3api/s3tables/handler_table.go | 8 + weed/s3api/s3tables/manager.go | 19 ++ weed/s3api/s3vectors/bind_bucket_test.go | 209 ++++++++++++++++++ weed/s3api/s3vectors/handlers.go | 55 +++-- 10 files changed, 464 insertions(+), 85 deletions(-) create mode 100644 weed/s3api/s3tables/handler_bind_bucket_test.go create mode 100644 weed/s3api/s3vectors/bind_bucket_test.go diff --git a/weed/s3api/s3tables/filer_ops.go b/weed/s3api/s3tables/filer_ops.go index 3775621a6a..80ad13ef25 100644 --- a/weed/s3api/s3tables/filer_ops.go +++ b/weed/s3api/s3tables/filer_ops.go @@ -483,14 +483,6 @@ func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.Seawe return resp.Entry, nil } -// deleteExtendedAttribute deletes an extended attribute from an entry -func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error { - return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error { - delete(extended, key) - return nil - }) -} - // deleteDirectory deletes a directory and all its contents func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error { dir, name := splitPath(path) @@ -772,6 +764,14 @@ func requireTableCreatedAt(extended map[string][]byte, path string, createdAt ti return nil } +// requireTableBucketCreatedAt is requireTableCreatedAt for a table bucket. +func requireTableBucketCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error { + if !isTableBucketCreatedAt(createdAt)(&filer_pb.Entry{Extended: extended}) { + return fmt.Errorf("%w: %s holds another table bucket", ErrConcurrentUpdate, path) + } + return nil +} + // confirmParent re-checks the parent after a create wrote written onto // childPath. If the parent is open, it clears the record's unfinished mark — // then checks the parent once more, since a delete claiming it in between diff --git a/weed/s3api/s3tables/handler_bind_bucket_test.go b/weed/s3api/s3tables/handler_bind_bucket_test.go new file mode 100644 index 0000000000..e36229a897 --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_bucket_test.go @@ -0,0 +1,104 @@ +package s3tables + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recreatedBucket is the record of the test bucket deleted and created again: +// another owner, and a creation time of its own. +func recreatedBucket(t *testing.T) map[string][]byte { + t.Helper() + meta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + return map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: meta, + } +} + +// bucketWriteOp is a write to the test bucket, with the attributes the bucket +// carries for it besides its record. +type bucketWriteOp struct { + operation string + input func(t *testing.T) interface{} + extra map[string][]byte +} + +func bucketWriteOps() []bucketWriteOp { + return []bucketWriteOp{ + {operation: "PutTableBucketPolicy", input: func(t *testing.T) interface{} { + return &PutTableBucketPolicyRequest{TableBucketARN: mustBucketARN(t), ResourcePolicy: bindTestPolicy} + }}, + {operation: "DeleteTableBucketPolicy", input: func(t *testing.T) interface{} { + return &DeleteTableBucketPolicyRequest{TableBucketARN: mustBucketARN(t)} + }, extra: map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}}, + {operation: "TagResource", input: func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: map[string]string{"k": "v"}} + }}, + {operation: "UntagResource", input: func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: mustBucketARN(t), TagKeys: []string{"k"}} + }, extra: map[string][]byte{ExtendedKeyTags: []byte(`{"k":"v","keep":"1"}`)}}, + {operation: "PutTableBucketMaintenanceConfiguration", input: func(t *testing.T) interface{} { + return &PutTableBucketMaintenanceConfigurationRequest{ + TableBucketARN: mustBucketARN(t), + Type: MaintenanceTypeIcebergUnreferencedFileRemoval, + Value: &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}, + } + }}, + } +} + +// A write to a bucket's policy, tags or maintenance settings is authorized +// against the bucket it read. If that bucket is deleted and another is created under the same name +// before the write, the values it compares can match the new bucket's just as +// well. The write checks the bucket by its creation time and is refused, and +// the new bucket is kept as it was created. +func TestBucketWritesSpareARecreatedBucket(t *testing.T) { + for _, op := range bucketWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + recreated := recreatedBucket(t) + for key, value := range op.extra { + extended[key] = value + recreated[key] = value + } + fs.Put(TablesPath, renameTestBucket, extended) + fs.BeforeUpdate = func() { fs.Put(TablesPath, renameTestBucket, recreated) } + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "the new bucket is left as it was created") + }) + } +} + +// The same writes still land on the bucket they were authorized against. +func TestBucketWritesLandOnTheAuthorizedBucket(t *testing.T) { + for _, op := range bucketWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + for key, value := range op.extra { + extended[key] = value + } + fs.Put(TablesPath, renameTestBucket, extended) + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + }) + } +} diff --git a/weed/s3api/s3tables/handler_bucket_get_list_delete.go b/weed/s3api/s3tables/handler_bucket_get_list_delete.go index 2e7c604988..34b8b8ab04 100644 --- a/weed/s3api/s3tables/handler_bucket_get_list_delete.go +++ b/weed/s3api/s3tables/handler_bucket_get_list_delete.go @@ -311,6 +311,9 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http if err := json.Unmarshal(data, &metadata); err != nil { return fmt.Errorf("failed to unmarshal metadata: %w", err) } + if otherTableBucket(r.Context(), metadata.CreatedAt) { + return filer_pb.ErrNotFound + } // Fetch bucket policy if it exists policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) diff --git a/weed/s3api/s3tables/handler_maintenance.go b/weed/s3api/s3tables/handler_maintenance.go index f359f0a38b..bd971e64cf 100644 --- a/weed/s3api/s3tables/handler_maintenance.go +++ b/weed/s3api/s3tables/handler_maintenance.go @@ -46,12 +46,13 @@ func (h *S3TablesHandler) handlePutTableBucketMaintenanceConfiguration(w http.Re return err } - if _, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName); err != nil { + _, createdAt, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName) + if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err } - if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value, nil); err != nil { + if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value, requireTableBucketCreatedAt, createdAt); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err } @@ -79,7 +80,7 @@ func (h *S3TablesHandler) handleGetTableBucketMaintenanceConfiguration(w http.Re return err } - bucketARN, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName) + bucketARN, _, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err @@ -124,10 +125,7 @@ func (h *S3TablesHandler) handlePutTableMaintenanceConfiguration(w http.Response } tablePath := GetTablePath(bucketName, namespaceName, tableName) - sameTable := func(extended map[string][]byte) error { - return requireTableCreatedAt(extended, tablePath, createdAt) - } - if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value, sameTable); err != nil { + if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value, requireTableCreatedAt, createdAt); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err } @@ -258,24 +256,24 @@ func buildJobStatusResponse(recorded MaintenanceJobStatus, config MaintenanceCon // putMaintenanceConfiguration merges one maintenance type into the stored // configuration, leaving the other types alone. The write checks the catalog // identity in the same conditional mutation, so it cannot land on a name a -// concurrent rename or delete has already soft-deleted. sameRecord, when set, -// refuses a record other than the one the caller was authorized against. +// concurrent rename or delete has already soft-deleted. sameRecord refuses a +// record other than the one created at createdAt, which the caller was +// authorized against. func (h *S3TablesHandler) putMaintenanceConfiguration( r *http.Request, filerClient FilerClient, resourcePath, maintenanceType string, value *MaintenanceConfigurationValue, - sameRecord func(extended map[string][]byte) error, + sameRecord func(extended map[string][]byte, path string, createdAt time.Time) error, + createdAt time.Time, ) error { return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { if len(extended[ExtendedKeyMetadata]) == 0 { return fmt.Errorf("%w: %s", filer_pb.ErrNotFound, resourcePath) } - if sameRecord != nil { - if err := sameRecord(extended); err != nil { - return err - } + if err := sameRecord(extended, resourcePath, createdAt); err != nil { + return err } config := MaintenanceConfiguration{} @@ -337,8 +335,9 @@ func (h *S3TablesHandler) parseTableTarget(tableBucketARN string, namespace []st } // authorizeMaintenanceBucket checks the caller may perform operation on the -// table bucket, returning the bucket ARN. -func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, error) { +// table bucket, returning the bucket ARN and the creation time of the bucket +// record the check read. +func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, time.Time, error) { bucketPath := GetTableBucketPath(bucketName) var bucketMetadata tableBucketMetadata @@ -355,7 +354,7 @@ func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClien return err }) if err != nil { - return "", err + return "", time.Time{}, err } bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) @@ -367,10 +366,10 @@ func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClien IdentityActions: getIdentityActions(r), DefaultAllow: h.defaultAllowFor(r), }) { - return "", NewAuthError(operation, principal, "not authorized to "+operation) + return "", time.Time{}, NewAuthError(operation, principal, "not authorized to "+operation) } - return bucketARN, nil + return bucketARN, bucketMetadata.CreatedAt, nil } // authorizeMaintenanceTable checks the caller may perform operation on the diff --git a/weed/s3api/s3tables/handler_namespace.go b/weed/s3api/s3tables/handler_namespace.go index ccd00298ba..4327fc2f56 100644 --- a/weed/s3api/s3tables/handler_namespace.go +++ b/weed/s3api/s3tables/handler_namespace.go @@ -151,6 +151,11 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return filer_pb.ErrNotFound + } + bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) principal := h.getAccountID(r) identityActions := getIdentityActions(r) diff --git a/weed/s3api/s3tables/handler_policy.go b/weed/s3api/s3tables/handler_policy.go index 25cf2fcd12..8483900637 100644 --- a/weed/s3api/s3tables/handler_policy.go +++ b/weed/s3api/s3tables/handler_policy.go @@ -12,24 +12,15 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) -// tableCreatedAtOf returns the creation time a table's metadata records, which -// tells the table apart from one created again under its name. -func tableCreatedAtOf(data []byte) (time.Time, error) { - var meta tableMetadataInternal - if err := json.Unmarshal(data, &meta); err != nil { - return time.Time{}, err - } - return meta.CreatedAt, nil -} - -// extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata. -// This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource, -// and handleUntagResource. +// extractResourceOwnerAndBucket extracts ownership info, bucket name and the +// creation time that tells the resource apart from one created again under its +// name. It consolidates the repeated pattern used in handleTagResource, +// handleListTagsForResource, and handleUntagResource. func (h *S3TablesHandler) extractResourceOwnerAndBucket( data []byte, resourcePath string, rType ResourceType, -) (ownerAccountID, bucketName string, err error) { +) (ownerAccountID, bucketName string, createdAt time.Time, err error) { // Extract bucket name from resource path (format: /buckets/{bucket}/... for both tables and buckets) parts := strings.Split(strings.Trim(resourcePath, "/"), "/") if len(parts) >= 2 { @@ -39,17 +30,17 @@ func (h *S3TablesHandler) extractResourceOwnerAndBucket( if rType == ResourceTypeTable { var meta tableMetadataInternal if err := json.Unmarshal(data, &meta); err != nil { - return "", "", err + return "", "", time.Time{}, err } - ownerAccountID = meta.OwnerAccountID + ownerAccountID, createdAt = meta.OwnerAccountID, meta.CreatedAt } else { var meta tableBucketMetadata if err := json.Unmarshal(data, &meta); err != nil { - return "", "", err + return "", "", time.Time{}, err } - ownerAccountID = meta.OwnerAccountID + ownerAccountID, createdAt = meta.OwnerAccountID, meta.CreatedAt } - return ownerAccountID, bucketName, nil + return ownerAccountID, bucketName, createdAt, nil } // handlePutTableBucketPolicy puts a policy on a table bucket @@ -133,6 +124,9 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h // landing mid-request sends the caller back through authorization. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateEntryExtended(r.Context(), client, bucketPath, func(extended map[string][]byte) error { + if err := requireTableBucketCreatedAt(extended, bucketPath, bucketMetadata.CreatedAt); err != nil { + return err + } if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(bucketPolicy)) { return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) } @@ -144,6 +138,8 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h if err != nil { if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table bucket policy") } @@ -299,11 +295,26 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.deleteExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) + return h.mutateEntryExtended(r.Context(), client, bucketPath, func(extended map[string][]byte) error { + if err := requireTableBucketCreatedAt(extended, bucketPath, bucketMetadata.CreatedAt); err != nil { + return err + } + if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(bucketPolicy)) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } + delete(extended, ExtendedKeyPolicy) + return nil + }) }) - if err != nil && !errors.Is(err, ErrAttributeNotFound) { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy") + if err != nil { + if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + } else { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy") + } return err } @@ -691,7 +702,7 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string - var tableCreatedAt time.Time + var createdAt time.Time requestTagKeys := mapKeys(req.Tags) err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check @@ -703,10 +714,11 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, resourceCreatedAt, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } + createdAt = resourceCreatedAt // Fetch bucket policy if we have a bucket name if bucketName != "" { @@ -721,9 +733,6 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque } if rType == ResourceTypeTable { - if tableCreatedAt, err = tableCreatedAtOf(data); err != nil { - return err - } if _, tableNamespace, tableName, err = parseTableFromARN(req.ResourceARN); err != nil { return err } @@ -796,12 +805,14 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") return fmt.Errorf("failed to marshal tags: %w", err) } + sameResource := requireTableCreatedAt + if rType != ResourceTypeTable { + sameResource = requireTableBucketCreatedAt + } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { - if rType == ResourceTypeTable { - if err := requireTableCreatedAt(extended, resourcePath, tableCreatedAt); err != nil { - return err - } + if err := sameResource(extended, resourcePath, createdAt); err != nil { + return err } extended[extendedKey] = tagsBytes return nil @@ -863,7 +874,7 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, _, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } @@ -976,7 +987,7 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string - var tableCreatedAt time.Time + var createdAt time.Time err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata) @@ -987,10 +998,11 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, resourceCreatedAt, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } + createdAt = resourceCreatedAt // Fetch bucket policy if we have a bucket name if bucketName != "" { @@ -1005,9 +1017,6 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req } if rType == ResourceTypeTable { - if tableCreatedAt, err = tableCreatedAtOf(data); err != nil { - return err - } if _, tableNamespace, tableName, err = parseTableFromARN(req.ResourceARN); err != nil { return err } @@ -1078,12 +1087,14 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") return fmt.Errorf("failed to marshal tags: %w", err) } + sameResource := requireTableCreatedAt + if rType != ResourceTypeTable { + sameResource = requireTableBucketCreatedAt + } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { - if rType == ResourceTypeTable { - if err := requireTableCreatedAt(extended, resourcePath, tableCreatedAt); err != nil { - return err - } + if err := sameResource(extended, resourcePath, createdAt); err != nil { + return err } extended[extendedKey] = tagsBytes return nil diff --git a/weed/s3api/s3tables/handler_table.go b/weed/s3api/s3tables/handler_table.go index afa5a2ddc9..6db8fab282 100644 --- a/weed/s3api/s3tables/handler_table.go +++ b/weed/s3api/s3tables/handler_table.go @@ -1009,6 +1009,10 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque } return err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return filer_pb.ErrNotFound + } tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) @@ -2233,6 +2237,10 @@ func (h *S3TablesHandler) authorizeCreateTable(w http.ResponseWriter, r *http.Re h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err)) return nil, err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return nil, filer_pb.ErrNotFound + } if !namespaceExists { // The implicit namespace has no record yet: authorize it as the diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index bffe3c9c5f..0334f80fec 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -76,6 +77,24 @@ func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext { return ctx } +type tableBucketCreatedAtKey struct{} + +// WithTableBucketCreatedAt binds a DeleteTableBucket, CreateNamespace, +// CreateTable or DeleteTable run through Execute to the bucket created at +// createdAt, so a +// caller that authorized the bucket itself never writes into one created +// again under its name. In-process only: no wire request can carry it. +func WithTableBucketCreatedAt(ctx context.Context, createdAt time.Time) context.Context { + return context.WithValue(ctx, tableBucketCreatedAtKey{}, createdAt) +} + +// otherTableBucket reports whether ctx binds the request to a bucket other +// than the one created at createdAt. +func otherTableBucket(ctx context.Context, createdAt time.Time) bool { + want, ok := ctx.Value(tableBucketCreatedAtKey{}).(time.Time) + return ok && !createdAt.Equal(want) +} + // Execute runs an S3 Tables operation and decodes the response into resp (if provided). func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error { body, err := json.Marshal(req) diff --git a/weed/s3api/s3vectors/bind_bucket_test.go b/weed/s3api/s3vectors/bind_bucket_test.go new file mode 100644 index 0000000000..3a73326874 --- /dev/null +++ b/weed/s3api/s3vectors/bind_bucket_test.go @@ -0,0 +1,209 @@ +package s3vectors + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +// A vector bucket policy write authorizes against the bucket it read: when the +// bucket is deleted and created again by another account before the write's +// compare-and-swap reads the entry, the write must not land on the new bucket. +func TestVectorBucketPolicyWritesKeepToAuthorizedBucket(t *testing.T) { + const policy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:GetVectorBucket","Resource":"*"}]}` + cases := []struct { + name string + call func(s *Server) *httptest.ResponseRecorder + }{ + {"put", func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}) + }}, + {"delete", func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handleDeleteVectorBucketPolicy, &DeleteVectorBucketPolicyRequest{VectorBucketName: "vec-bucket"}) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}); w.Code != http.StatusOK { + t.Fatalf("seed policy: %d %s", w.Code, w.Body.String()) + } + + dir, name := splitEntryPath(s3tables.GetTableBucketPath("vec-bucket")) + lookups := 0 + fs.OnLookup = func(d, n string) { + if d == dir && n == name { + lookups++ + } + } + if w := tc.call(s); w.Code != http.StatusOK { + t.Fatalf("baseline %s: %d %s", tc.name, w.Code, w.Body.String()) + } + fs.OnLookup = nil + if w := postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}); w.Code != http.StatusOK { + t.Fatalf("reseed policy: %d %s", w.Code, w.Body.String()) + } + // The last bucket lookup is the compare-and-swap read; replace the + // bucket with another account's right before it. + last, seen := lookups, 0 + fs.OnLookup = func(d, n string) { + if d != dir || n != name { + return + } + if seen++; seen != last { + return + } + entry := fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Errorf("decode bucket metadata: %v", err) + return + } + meta["ownerAccountId"] = "bob" + meta["createdAt"] = time.Now().Add(time.Hour).UTC() + data, _ := json.Marshal(meta) + ext := make(map[string][]byte, len(entry.Extended)) + for k, v := range entry.Extended { + ext[k] = v + } + ext[s3tables.ExtendedKeyMetadata] = data + ext[vectorPolicyKey] = []byte("bob-policy") + fs.Put(dir, name, ext) + } + w := tc.call(s) + fs.OnLookup = nil + if w.Code != http.StatusConflict { + t.Fatalf("%s over a replaced bucket = %d %s, want 409", tc.name, w.Code, w.Body.String()) + } + if got := string(fs.Get(dir, name).Extended[vectorPolicyKey]); got != "bob-policy" { + t.Fatalf("%s rewrote the replacing bucket's policy: %q", tc.name, got) + } + }) + } +} + +// replaceBucket swaps the bucket's record for another account's bucket of the +// same name, keeping its contents. +func replaceBucket(t *testing.T, fs *s3tablestest.MemFiler, bucket string) { + dir, name := splitEntryPath(s3tables.GetTableBucketPath(bucket)) + entry := fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Errorf("decode bucket metadata: %v", err) + return + } + meta["ownerAccountId"] = "bob" + meta["createdAt"] = time.Now().Add(time.Hour).UTC() + data, _ := json.Marshal(meta) + ext := make(map[string][]byte, len(entry.Extended)) + for k, v := range entry.Extended { + ext[k] = v + } + ext[s3tables.ExtendedKeyMetadata] = data + fs.Put(dir, name, ext) +} + +// replaceBucketAfterAuthorization replaces bucket right after the lookups +// authorizeVector makes, counted on a probe bucket of the same shape. +func replaceBucketAfterAuthorization(t *testing.T, fs *s3tablestest.MemFiler, s *Server, bucket string) { + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-probe"}); w.Code != http.StatusOK { + t.Fatalf("create probe bucket: %d %s", w.Code, w.Body.String()) + } + probeDir, probeName := splitEntryPath(s3tables.GetTableBucketPath("vec-probe")) + authLookups := 0 + fs.OnLookup = func(d, n string) { + if d == probeDir && n == probeName { + authLookups++ + } + } + probe := httptest.NewRequest(http.MethodPost, "/", nil) + probe = probe.WithContext(contextWithIdentity(probe.Context(), "")) + if _, err := s.authorizeVector(probe.Context(), probe, "s3vectors:DeleteVectorBucket", "vec-probe", ""); err != nil { + t.Fatalf("authorize probe: %v", err) + } + + dir, name := splitEntryPath(s3tables.GetTableBucketPath(bucket)) + seen := 0 + fs.OnLookup = func(d, n string) { + if d == dir && n == name { + if seen++; seen == authLookups+1 { + replaceBucket(t, fs, bucket) + } + } + } +} + +// DeleteVectorBucket deletes only the bucket it authorized: one deleted and +// created again by another account in between survives. +func TestDeleteVectorBucketKeepsReplacingBucket(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := postJSONAs(t, "", s.handleDeleteVectorBucket, &DeleteVectorBucketRequest{VectorBucketName: "vec-bucket"}) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("delete over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + dir, name := splitEntryPath(s3tables.GetTableBucketPath("vec-bucket")) + if fs.Get(dir, name) == nil { + t.Fatal("DeleteVectorBucket removed the bucket that replaced the authorized one") + } +} + +// CreateIndex and DeleteIndex write only into the bucket they authorized. +func TestIndexWritesKeepToAuthorizedBucket(t *testing.T) { + newServer := func(t *testing.T) (*s3tablestest.MemFiler, *Server) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + return fs, s + } + createIndex := func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }) + } + indexDir, indexName := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx")) + + t.Run("create", func(t *testing.T) { + fs, s := newServer(t) + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := createIndex(s) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("create over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + if fs.Get(indexDir, indexName) != nil { + t.Fatal("CreateIndex wrote an index into the bucket that replaced the authorized one") + } + }) + t.Run("delete", func(t *testing.T) { + fs, s := newServer(t) + if w := createIndex(s); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := postJSONAs(t, "", s.handleDeleteIndex, &DeleteIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"}) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("delete over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + if fs.Get(indexDir, indexName) == nil { + t.Fatal("DeleteIndex removed an index of the bucket that replaced the authorized one") + } + }) +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 7e7bd12e83..af762d7097 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -405,12 +405,16 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteVectorBucket", arns.bucket, ""); err != nil { + record, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteVectorBucket", arns.bucket, "") + if err != nil { catalogErr(w, err) return } + // Only the bucket authorized above is deleted, not one created again + // under its name since. + ctx := s3tables.WithTableBucketCreatedAt(r.Context(), record.CreatedAt) var resp map[string]interface{} - err = s.internalCatalog(r.Context(), "DeleteTableBucket", + err = s.internalCatalog(ctx, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: s.s3tablesArn(arns.bucket)}, &resp) if err != nil && catalogErrCode(err) == s3tables.ErrCodeBucketNotEmpty { // The implicit index namespace counts as bucket content even when it @@ -423,7 +427,7 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request Namespace: []string{indexNamespace}, }, &nsResp) if nsErr == nil { - err = s.internalCatalog(r.Context(), "DeleteTableBucket", + err = s.internalCatalog(ctx, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: s.s3tablesArn(arns.bucket)}, &resp) } else if code := catalogErrCode(nsErr); code != s3tables.ErrCodeNoSuchNamespace { glog.V(1).Infof("S3 Vectors: drop internal namespace in %s: %v", arns.bucket, nsErr) @@ -752,14 +756,26 @@ func isVectorBucket(record *s3tables.GetTableBucketResponse) bool { // mutation; catalogErr renders it as AccessDenied. var errVectorDenied = &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"} +var errVectorBucketReplaced = &s3tables.S3TablesError{Type: s3tables.ErrCodeConflict, + Message: "vector bucket changed concurrently, retry the request"} + // checkVectorPolicyDoc evaluates the caller's vector action against the // document the entry currently holds. Called inside a policy mutation's // compare-and-swap, so a grant revoked between reads never authorizes the -// pending write. -func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket, owner string, ext map[string][]byte) bool { +// pending write, and the entry must still be the bucket record was read from. +func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket string, record *s3tables.GetTableBucketResponse, ext map[string][]byte) error { + var meta struct { + CreatedAt time.Time `json:"createdAt"` + } + if json.Unmarshal(ext[s3tables.ExtendedKeyMetadata], &meta) != nil || !meta.CreatedAt.Equal(record.CreatedAt) { + return errVectorBucketReplaced + } principal := s.tables.PrincipalFor(r) - return s3tables.CheckPermissionWithContext(action, principal, owner, - vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r, bucket)) + if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, + vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r, bucket)) { + return errVectorDenied + } + return nil } // authorizeVector grants the request's own vector operation — not a catalog @@ -818,8 +834,8 @@ func (s *Server) handlePutVectorBucketPolicy(w http.ResponseWriter, r *http.Requ return } err = s.mutateVectorPolicy(r.Context(), arns.bucket, func(ext map[string][]byte) error { - if !s.checkVectorPolicyDoc(r, "s3vectors:PutVectorBucketPolicy", arns.bucket, record.OwnerAccountID, ext) { - return errVectorDenied + if err := s.checkVectorPolicyDoc(r, "s3vectors:PutVectorBucketPolicy", arns.bucket, record, ext); err != nil { + return err } // A raw vector document still in the shared slot migrates out with // this write; catalog rules in it stay untouched. @@ -889,8 +905,8 @@ func (s *Server) handleDeleteVectorBucketPolicy(w http.ResponseWriter, r *http.R return } err = s.mutateVectorPolicy(r.Context(), arns.bucket, func(ext map[string][]byte) error { - if !s.checkVectorPolicyDoc(r, "s3vectors:DeleteVectorBucketPolicy", arns.bucket, record.OwnerAccountID, ext) { - return errVectorDenied + if err := s.checkVectorPolicyDoc(r, "s3vectors:DeleteVectorBucketPolicy", arns.bucket, record, ext); err != nil { + return err } removed := false if _, ok := ext[vectorPolicyKey]; ok { @@ -949,12 +965,15 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { } // The create grant is exercised before any write: a denied caller must not // leave even the implicit namespace behind. - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:CreateIndex", arns.bucket, arns.index); err != nil { + bucketRecord, err := s.authorizeVector(r.Context(), r, "s3vectors:CreateIndex", arns.bucket, arns.index) + if err != nil { catalogErr(w, err) return } + // The catalog writes land only in the bucket authorized above. + bound := s3tables.WithTableBucketCreatedAt(r.Context(), bucketRecord.CreatedAt) // The namespace is implicit on AWS and internal bookkeeping here. - if err := s.ensureIndexNamespace(r.Context(), arns.bucket); err != nil { + if err := s.ensureIndexNamespace(bound, arns.bucket); err != nil { catalogErr(w, err) return } @@ -989,7 +1008,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { } } metadataJSON, _ := json.Marshal(metadata) - err = s.internalCatalog(r.Context(), "CreateTable", &s3tables.CreateTableRequest{ + err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, Name: arns.index, @@ -1189,7 +1208,8 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteIndex", arns.bucket, arns.index); err != nil { + bucketRecord, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteIndex", arns.bucket, arns.index) + if err != nil { catalogErr(w, err) return } @@ -1213,8 +1233,9 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { // Deleting the catalog record is the delete: the catalog purges the // dataset under the index's location along with the entry. var resp map[string]interface{} - // The token limits the delete to the record checked above. - err = s.internalCatalog(r.Context(), "DeleteTable", &s3tables.DeleteTableRequest{ + // The token limits the delete to the record checked above, and the bound + // bucket to the one authorized. + err = s.internalCatalog(s3tables.WithTableBucketCreatedAt(r.Context(), bucketRecord.CreatedAt), "DeleteTable", &s3tables.DeleteTableRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, Name: arns.index, From aa2d94cc50b8e8da8bc3368d3e888c68d5c26f1f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 03:21:26 -0800 Subject: [PATCH 22/31] ci: cancel a PR's runs when it is merged or closed (#12330) A merged or closed PR kept every pull_request run going: concurrency groups cancel a run only when a newer push arrives. After #12329 merged, 16 of its runs were still queued or running. On pull_request_target closed, cancel the queued and running pull_request runs on the PR's head branch whose commit belongs to the PR. Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK Co-authored-by: Claude Opus 5.5 --- .github/workflows/cancel-closed-pr-runs.yml | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/cancel-closed-pr-runs.yml diff --git a/.github/workflows/cancel-closed-pr-runs.yml b/.github/workflows/cancel-closed-pr-runs.yml new file mode 100644 index 0000000000..0e07039ac9 --- /dev/null +++ b/.github/workflows/cancel-closed-pr-runs.yml @@ -0,0 +1,37 @@ +name: "Cancel runs of closed PRs" + +# Merging or closing a PR leaves its pull_request runs going: concurrency +# groups cancel a run only when a newer push arrives. pull_request_target gives +# this job a token that can cancel runs, fork PRs included. It checks out and +# runs no PR code. +on: + pull_request_target: + types: [closed] + +permissions: + actions: write + pull-requests: read + +jobs: + cancel: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Cancel the PR's queued and running pull_request runs + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + set -euo pipefail + # Match the PR's own commits: forks reuse branch names such as master. + shas=$(gh api --paginate "repos/$GH_REPO/pulls/$PR/commits" --jq '.[].sha') + for status in queued in_progress waiting pending requested; do + gh run list --branch "$HEAD_REF" --event pull_request --status "$status" \ + --limit 200 --json databaseId,headSha --jq '.[] | "\(.databaseId) \(.headSha)"' + done | while read -r id sha; do + grep -qx "$sha" <<<"$shas" || continue + echo "cancel run $id at $sha" + gh run cancel "$id" || true + done From 984eac6a790be89e2a11de175a7413e1e3e32dd3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:24:06 +0000 Subject: [PATCH 23/31] 4.52 --- k8s/charts/seaweedfs/Chart.yaml | 4 ++-- weed/util/version/constants.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 357c1fb96f..68b529b13e 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "4.51" +appVersion: "4.52" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.51.0 +version: 4.52.0 diff --git a/weed/util/version/constants.go b/weed/util/version/constants.go index 2e3c4c5468..0cf26abbe7 100644 --- a/weed/util/version/constants.go +++ b/weed/util/version/constants.go @@ -9,7 +9,7 @@ import ( var ( MAJOR_VERSION = int32(4) - MINOR_VERSION = int32(51) + MINOR_VERSION = int32(52) VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION) VERSION = util.SizeLimit + " " + VERSION_NUMBER COMMIT = "" From 7298b4ed37c887dcb43ba9ccbb12e386beb6cc5f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 09:55:09 -0800 Subject: [PATCH 24/31] fix(s3tables,s3vectors): bind DeleteNamespace and the vector bucket cleanup to the authorized records (#12331) * s3tables: bind DeleteNamespace to the authorized namespace The delete claimed whatever finished namespace held the name at claim time, so one deleted and created again by another owner after the authorization read was removed under the first one's authorization. The claim now requires the creation time that was authorized. Co-Authored-By: Claude Opus 5.5 * s3vectors: bind DeleteVectorBucket's namespace cleanup to the authorized bucket After the bound DeleteTableBucket reported the bucket not empty, the index namespace cleanup ran unbound, so a bucket deleted and created again in between lost its empty index namespace under the first bucket's authorization. DeleteNamespace now honors the bucket binding, checking the bucket after reading the namespace, and the cleanup carries it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../s3tables/handler_bind_authorized_test.go | 25 ++++++++++++++ weed/s3api/s3tables/handler_namespace.go | 14 +++++++- weed/s3api/s3tables/manager.go | 16 ++++++--- weed/s3api/s3vectors/bind_bucket_test.go | 34 +++++++++++++++++++ weed/s3api/s3vectors/handlers.go | 2 +- 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/weed/s3api/s3tables/handler_bind_authorized_test.go b/weed/s3api/s3tables/handler_bind_authorized_test.go index ce48bdf3e1..0544ec5406 100644 --- a/weed/s3api/s3tables/handler_bind_authorized_test.go +++ b/weed/s3api/s3tables/handler_bind_authorized_test.go @@ -583,3 +583,28 @@ func TestUpdateNamespaceSparesARecreatedNamespace(t *testing.T) { assert.Equal(t, http.StatusConflict, code, "%+v", resp) assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "the new namespace is untouched") } + +// DeleteNamespace removes only the namespace it was authorized against: one +// deleted and created again before the claim reads as the first one gone. +func TestDeleteNamespaceSparesARecreatedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnUpdate = nil + fs.Put(bucketPath, "ns", recreated) + } + + code, resp := runHandler(t, m, fs.Client, "DeleteNamespace", &DeleteNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the new namespace survives") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "the new namespace is untouched") +} diff --git a/weed/s3api/s3tables/handler_namespace.go b/weed/s3api/s3tables/handler_namespace.go index 4327fc2f56..f693ce833e 100644 --- a/weed/s3api/s3tables/handler_namespace.go +++ b/weed/s3api/s3tables/handler_namespace.go @@ -757,6 +757,17 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R if err := json.Unmarshal(data, &metadata); err != nil { return fmt.Errorf("failed to unmarshal metadata: %w", err) } + // Checked after the namespace read: a bucket created again holds only + // namespaces of its own, and the claim below is bound to this one. + if createdAt, bound := boundTableBucket(r.Context()); bound { + bucket, err := h.lookupEntry(r.Context(), client, bucketPath) + if err != nil { + return err + } + if !isTableBucketCreatedAt(createdAt)(bucket) { + return filer_pb.ErrNotFound + } + } // Fetch bucket policy if it exists policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) @@ -802,11 +813,12 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R // A namespace whose create has not confirmed it is absent, and claiming it // would replace the create's mark: a refused delete then lifts the claim and // leaves the namespace looking confirmed before its create checked the bucket. + // One created again since it was authorized is absent too. var claim string var claimed map[string][]byte err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { var claimErr error - claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isFinishedNamespaceEntry) + claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isNamespaceCreatedAt(metadata.CreatedAt)) return claimErr }) if err != nil { diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index 0334f80fec..e3be966476 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -80,18 +80,24 @@ func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext { type tableBucketCreatedAtKey struct{} // WithTableBucketCreatedAt binds a DeleteTableBucket, CreateNamespace, -// CreateTable or DeleteTable run through Execute to the bucket created at -// createdAt, so a -// caller that authorized the bucket itself never writes into one created -// again under its name. In-process only: no wire request can carry it. +// DeleteNamespace, CreateTable or DeleteTable run through Execute to the +// bucket created at createdAt, so a caller that authorized the bucket itself +// never writes into one created again under its name. In-process only: no +// wire request can carry it. func WithTableBucketCreatedAt(ctx context.Context, createdAt time.Time) context.Context { return context.WithValue(ctx, tableBucketCreatedAtKey{}, createdAt) } +// boundTableBucket returns the bucket creation time ctx binds the request to. +func boundTableBucket(ctx context.Context) (time.Time, bool) { + createdAt, ok := ctx.Value(tableBucketCreatedAtKey{}).(time.Time) + return createdAt, ok +} + // otherTableBucket reports whether ctx binds the request to a bucket other // than the one created at createdAt. func otherTableBucket(ctx context.Context, createdAt time.Time) bool { - want, ok := ctx.Value(tableBucketCreatedAtKey{}).(time.Time) + want, ok := boundTableBucket(ctx) return ok && !createdAt.Equal(want) } diff --git a/weed/s3api/s3vectors/bind_bucket_test.go b/weed/s3api/s3vectors/bind_bucket_test.go index 3a73326874..b70a0cba36 100644 --- a/weed/s3api/s3vectors/bind_bucket_test.go +++ b/weed/s3api/s3vectors/bind_bucket_test.go @@ -207,3 +207,37 @@ func TestIndexWritesKeepToAuthorizedBucket(t *testing.T) { } }) } + +// DeleteVectorBucket drops the empty index namespace only from the bucket it +// authorized: a bucket created again before that cleanup keeps its own. +func TestDeleteVectorBucketCleanupKeepsReplacingBucket(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleDeleteIndex, &DeleteIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"}); w.Code != http.StatusOK { + t.Fatalf("delete index: %d %s", w.Code, w.Body.String()) + } + + bucketPath := s3tables.GetTableBucketPath("vec-bucket") + fs.OnLookup = func(d, n string) { + if d == bucketPath && n == indexNamespace { + fs.OnLookup = nil + replaceBucket(t, fs, "vec-bucket") + } + } + w := postJSONAs(t, "", s.handleDeleteVectorBucket, &DeleteVectorBucketRequest{VectorBucketName: "vec-bucket"}) + fs.OnLookup = nil + if w.Code == http.StatusOK { + t.Fatalf("delete over a replaced bucket succeeded: %s", w.Body.String()) + } + if fs.Get(bucketPath, indexNamespace) == nil { + t.Fatal("DeleteVectorBucket removed the index namespace of the bucket that replaced the authorized one") + } +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index af762d7097..9f9be68b51 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -421,7 +421,7 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request // holds no indexes; drop it under the delete's own authorization and // retry once. A namespace still holding indexes keeps the conflict. var nsResp map[string]interface{} - nsErr := s.internalCatalog(context.WithoutCancel(r.Context()), "DeleteNamespace", + nsErr := s.internalCatalog(context.WithoutCancel(ctx), "DeleteNamespace", &s3tables.DeleteNamespaceRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, From 1637941368b3f0497a473591abae4d993c0e7361 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Mon, 2 Nov 2026 19:09:22 +0100 Subject: [PATCH 25/31] filer: allow renaming into a sibling that shares the name prefix (#12337) CanRename rejected a move whenever the new parent directory's path started with the source path as a plain string, so moving /dir/a into /dir/a.d (or anything under /dir/ab) failed with "can not move directory to a subdirectory of itself". Check the path boundary instead: reject only when the new parent is the source itself or lies under it. Claude-Session: https://claude.ai/code/session_01VncHKTehVroSAzRMUHykEX Co-authored-by: Claude Opus 5.5 --- weed/filer/filer_rename.go | 9 ++++++++- weed/filer/filer_rename_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/weed/filer/filer_rename.go b/weed/filer/filer_rename.go index b39d9872e1..1a9c48fc59 100644 --- a/weed/filer/filer_rename.go +++ b/weed/filer/filer_rename.go @@ -10,7 +10,7 @@ import ( func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error { sourcePath := source.Child(oldName) - if strings.HasPrefix(string(target), string(sourcePath)) { + if movesIntoItself(sourcePath, target) { return fmt.Errorf("mv: can not move directory to a subdirectory of itself") } @@ -33,6 +33,13 @@ func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, old return nil } +// movesIntoItself reports whether target, the new parent directory, is +// sourcePath or lies under it. A sibling that only shares a name prefix, +// such as /a.d for /a, does not. +func movesIntoItself(sourcePath, target util.FullPath) bool { + return target == sourcePath || target.IsUnder(sourcePath) +} + func (f *Filer) DetectBucket(source util.FullPath) (bucket string) { if strings.HasPrefix(string(source), f.DirBucketsPath+"/") { bucketAndObjectKey := string(source)[len(f.DirBucketsPath)+1:] diff --git a/weed/filer/filer_rename_test.go b/weed/filer/filer_rename_test.go index d7096963fa..5df9429c14 100644 --- a/weed/filer/filer_rename_test.go +++ b/weed/filer/filer_rename_test.go @@ -29,3 +29,27 @@ func TestDetectBucket(t *testing.T) { }) } } + +func TestMovesIntoItself(t *testing.T) { + tests := []struct { + name string + sourcePath string + target string + want bool + }{ + {"into itself", "/dir/a", "/dir/a", true}, + {"into a child", "/dir/a", "/dir/a/b", true}, + {"into a deeper descendant", "/dir/a", "/dir/a/b/c", true}, + {"into a sibling sharing the name prefix", "/dir/a", "/dir/a.d", false}, + {"into a descendant of that sibling", "/dir/a", "/dir/ab/c", false}, + {"into the parent", "/dir/a", "/dir", false}, + {"into an unrelated directory", "/dir/a", "/other", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := movesIntoItself(util.FullPath(tt.sourcePath), util.FullPath(tt.target)); got != tt.want { + t.Errorf("movesIntoItself(%q, %q) = %v, want %v", tt.sourcePath, tt.target, got, tt.want) + } + }) + } +} From d650abe0e76acda59728f3263c314c65d8ae31e5 Mon Sep 17 00:00:00 2001 From: Pat White Date: Mon, 2 Nov 2026 10:15:01 -0800 Subject: [PATCH 26/31] compression: compress JSON uploads, and match file types by extension (#12339) * compression: compress JSON uploads, and match file types by extension IsCompressableFileType did not recognise JSON media types, so S3 uploads of application/json (which carry no file name) were never compressed. Its callers also passed filepath.Base of the file name where it expects the extension, so the .json/.js/.html/... extension rules never matched a real file name. Both callers now pass filepath.Ext; the function lower-cases the extension and media type, ignores media-type parameters ("application/json; charset=utf-8"), and treats */json and *+json types as compressible. * compression: treat application/json-seq as JSON Co-Authored-By: Claude Opus 5.5 * volume(rust): compress JSON uploads and match media types without parameters Co-Authored-By: Claude Opus 5.5 * operation: keep uploader gzip only when it saves at least 10% Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Pat White <5015863+patwhite@users.noreply.github.com> Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/server/handlers.rs | 37 +++++++++++++++++----- weed/operation/needle_parse_test.go | 28 ++++++++++++++++ weed/operation/upload_content.go | 4 +-- weed/storage/needle/needle_parse_upload.go | 2 +- weed/util/compression.go | 10 ++++++ weed/util/compression_test.go | 31 ++++++++++++++++++ 6 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 weed/util/compression_test.go diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index d2d02d35b0..91c2d2a7ca 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -3101,15 +3101,12 @@ pub async fn post_handler( // Auto-compress compressible file types (matches Go's IsCompressableFileType). // Only compress if not already gzipped and compression saves >10%. - // Go uses filepath.Base(pu.FileName) for extension detection (not the URL path). let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest { - let ext = { - let dot_pos = filename.rfind('.'); - dot_pos - .map(|p| filename[p..].to_lowercase()) - .unwrap_or_default() - }; - if is_compressible_file_type(&ext, &mime_type) { + let ext = filename + .rfind('.') + .map(|p| &filename[p..]) + .unwrap_or_default(); + if is_compressible_file_type(ext, &mime_type) { if let Some(compressed) = try_gzip_data(&body_data) { if compressed.len() * 10 < body_data.len() * 9 { (compressed, true) @@ -4728,6 +4725,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option { /// Check if a file type should be compressed based on extension and MIME type. /// Returns true only when we are sure the type is compressible. fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { + let ext = ext.to_ascii_lowercase(); + let ext = ext.as_str(); + let mtype = mtype + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + let mtype = mtype.as_str(); // text/* if mtype.starts_with("text/") { return true; @@ -4760,6 +4766,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { if mtype.ends_with("script") { return true; } + if mtype.ends_with("json") || mtype.ends_with("json-seq") { + return true; + } if mtype.ends_with("vnd.rar") { return false; } @@ -5245,6 +5254,18 @@ mod tests { assert!(is_compressible_file_type("", "application/javascript")); assert!(!is_compressible_file_type("", "application/zstd")); assert!(!is_compressible_file_type("", "application/vnd.rar")); + assert!(is_compressible_file_type("", "application/json")); + assert!(is_compressible_file_type( + "", + "application/json; charset=utf-8" + )); + assert!(is_compressible_file_type("", "Application/JSON")); + assert!(is_compressible_file_type("", "application/ld+json")); + assert!(is_compressible_file_type("", "application/x-ndjson")); + assert!(is_compressible_file_type("", "application/json-seq")); + assert!(is_compressible_file_type("", "text/plain; charset=utf-8")); + assert!(is_compressible_file_type(".JSON", "")); + assert!(!is_compressible_file_type(".gz", "application/json")); // Audio assert!(is_compressible_file_type(".wav", "audio/wav")); diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index 339d4507e2..fb1d530a78 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "math/rand" "net/http" "testing" @@ -118,6 +119,33 @@ func TestCreateNeedleFromRequest(t *testing.T) { } +func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) { + data := make([]byte, 4300) + rand.New(rand.NewSource(1)).Read(data[:4000]) + gzipped, _ := util.GzipData(data) + if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 { + t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data)) + } + + mockClient := &MockClient{} + var stored *needle.Needle + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { + assert.Equal(t, nil, err, "upload: %v", err) + stored = n + } + uploadOption := &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0", + Filename: "doc.pdf", + MimeType: "application/pdf", + } + newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption) + if stored == nil { + t.Fatal("no needle reached the volume server") + } + assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed") + assert.Equal(t, data, stored.Data) +} + var textContent = `Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index a44dc38e8e..4d829e33e7 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option option.MimeType = "" } } - if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { + if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { shouldGzipNow = true } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 { var compressed []byte @@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option compressed, compressErr := util.GzipData(data) // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) if compressErr == nil { - if len(compressed) < len(data) { + if len(compressed)*10 < len(data)*9 { data = compressed contentIsGzipped = true } diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 0707acef14..3a668de9b6 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -96,7 +96,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p } } else if r.FormValue("type") != "replicate" { // replica writes must keep the source needle's compression state, not re-derive it - ext := filepath.Base(pu.FileName) + ext := filepath.Ext(pu.FileName) mimeType := pu.MimeType if mimeType == "" { mimeType = http.DetectContentType(pu.Data) diff --git a/weed/util/compression.go b/weed/util/compression.go index d62ba90883..aac6b1e012 100644 --- a/weed/util/compression.go +++ b/weed/util/compression.go @@ -99,6 +99,12 @@ func IsZstdContent(data []byte) bool { /* * Default not to compressed since compression can be done on client side. */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) { + ext = strings.ToLower(ext) + // Match on the media type alone: "application/json; charset=utf-8" is JSON. + if i := strings.IndexByte(mtype, ';'); i >= 0 { + mtype = strings.TrimSpace(mtype[:i]) + } + mtype = strings.ToLower(mtype) // text if strings.HasPrefix(mtype, "text/") { @@ -137,6 +143,10 @@ func IsZstdContent(data []byte) bool { if strings.HasSuffix(mtype, "script") { return true, true } + // application/json, application/ld+json, application/x-ndjson, application/json-seq, ... + if strings.HasSuffix(mtype, "json") || strings.HasSuffix(mtype, "json-seq") { + return true, true + } if strings.HasSuffix(mtype, "vnd.rar") { return false, true } diff --git a/weed/util/compression_test.go b/weed/util/compression_test.go new file mode 100644 index 0000000000..e9486be3e1 --- /dev/null +++ b/weed/util/compression_test.go @@ -0,0 +1,31 @@ +package util + +import "testing" + +func TestIsCompressableFileType(t *testing.T) { + tests := []struct { + ext, mtype string + compress, isSure bool + }{ + {".json", "", true, true}, + {".JSON", "", true, true}, + {"", "application/json", true, true}, + {"", "application/json; charset=utf-8", true, true}, + {"", "application/ld+json", true, true}, + {"", "application/x-ndjson", true, true}, + {"", "application/json-seq", true, true}, + {"", "Application/JSON", true, true}, + {"", "text/plain; charset=utf-8", true, true}, + {"", "application/xml", true, true}, + {".gz", "application/json", false, true}, + {"", "application/gzip", false, false}, + {"", "image/png", false, true}, + {"", "", false, false}, + } + for _, tt := range tests { + compress, isSure := IsCompressableFileType(tt.ext, tt.mtype) + if compress != tt.compress || isSure != tt.isSure { + t.Errorf("IsCompressableFileType(%q, %q) = (%v, %v), want (%v, %v)", tt.ext, tt.mtype, compress, isSure, tt.compress, tt.isSure) + } + } +} From 690aabda0e1715ae0aed1e8b4a17b9b6e9651528 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 10:28:48 -0800 Subject: [PATCH 27/31] filer: reuse IsEqualOrUnder for the rename subdirectory check (#12338) * filer: test the rename subdirectory check through CanRename Co-Authored-By: Claude Opus 5.5 * filer: reuse IsEqualOrUnder for the rename subdirectory check A trailing slash on the old name made the source path end in a slash, so a child of the source was no longer seen as under it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/filer/filer_rename.go | 9 +------ weed/filer/filer_rename_test.go | 43 ++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/weed/filer/filer_rename.go b/weed/filer/filer_rename.go index 1a9c48fc59..7cfd35e88f 100644 --- a/weed/filer/filer_rename.go +++ b/weed/filer/filer_rename.go @@ -10,7 +10,7 @@ import ( func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error { sourcePath := source.Child(oldName) - if movesIntoItself(sourcePath, target) { + if util.IsEqualOrUnder(string(target), string(sourcePath)) { return fmt.Errorf("mv: can not move directory to a subdirectory of itself") } @@ -33,13 +33,6 @@ func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, old return nil } -// movesIntoItself reports whether target, the new parent directory, is -// sourcePath or lies under it. A sibling that only shares a name prefix, -// such as /a.d for /a, does not. -func movesIntoItself(sourcePath, target util.FullPath) bool { - return target == sourcePath || target.IsUnder(sourcePath) -} - func (f *Filer) DetectBucket(source util.FullPath) (bucket string) { if strings.HasPrefix(string(source), f.DirBucketsPath+"/") { bucketAndObjectKey := string(source)[len(f.DirBucketsPath)+1:] diff --git a/weed/filer/filer_rename_test.go b/weed/filer/filer_rename_test.go index 5df9429c14..b5447c7539 100644 --- a/weed/filer/filer_rename_test.go +++ b/weed/filer/filer_rename_test.go @@ -1,6 +1,9 @@ package filer import ( + "context" + "os" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/util" @@ -30,25 +33,37 @@ func TestDetectBucket(t *testing.T) { } } -func TestMovesIntoItself(t *testing.T) { +func TestCanRenameIntoItself(t *testing.T) { + f, store := newTestFilerWithStubStore(t) + f.DirBucketsPath = "/buckets" + if err := store.InsertEntry(context.Background(), &Entry{FullPath: "/dir/a", Attr: Attr{Mode: os.ModeDir}}); err != nil { + t.Fatal(err) + } tests := []struct { - name string - sourcePath string - target string - want bool + name string + oldName string + target string + reject bool }{ - {"into itself", "/dir/a", "/dir/a", true}, - {"into a child", "/dir/a", "/dir/a/b", true}, - {"into a deeper descendant", "/dir/a", "/dir/a/b/c", true}, - {"into a sibling sharing the name prefix", "/dir/a", "/dir/a.d", false}, - {"into a descendant of that sibling", "/dir/a", "/dir/ab/c", false}, - {"into the parent", "/dir/a", "/dir", false}, - {"into an unrelated directory", "/dir/a", "/other", false}, + {"into itself", "a", "/dir/a", true}, + {"into a child", "a", "/dir/a/b", true}, + {"into a deeper descendant", "a", "/dir/a/b/c", true}, + {"into a child with a trailing slash", "a", "/dir/a/b/", true}, + {"old name with a trailing slash into a child", "a/", "/dir/a/b", true}, + {"into a sibling sharing the name prefix", "a", "/dir/a.d", false}, + {"into a descendant of that sibling", "a", "/dir/ab/c", false}, + {"into the parent", "a", "/dir", false}, + {"into an unrelated directory", "a", "/other", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := movesIntoItself(util.FullPath(tt.sourcePath), util.FullPath(tt.target)); got != tt.want { - t.Errorf("movesIntoItself(%q, %q) = %v, want %v", tt.sourcePath, tt.target, got, tt.want) + err := f.CanRename(context.Background(), "/dir", util.FullPath(tt.target), tt.oldName) + rejected := err != nil && strings.Contains(err.Error(), "subdirectory of itself") + if rejected != tt.reject { + t.Errorf("CanRename(/dir, %q, %q) = %v, want rejected=%v", tt.target, tt.oldName, err, tt.reject) + } + if !tt.reject && err != nil { + t.Errorf("CanRename(/dir, %q, %q) = %v, want nil", tt.target, tt.oldName, err) } }) } From ec968aa0165ce0df0959fd4482b61d50e2eb006e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 10:33:26 -0800 Subject: [PATCH 28/31] s3vectors: support TagResource, UntagResource and ListTagsForResource (#12335) * s3vectors: tag vector buckets and indexes TagResource, UntagResource and ListTagsForResource answer on /tags/{resourceArn} (POST, DELETE, GET) and by name. Each authorizes its own s3vectors action on the bucket or index ARN, then reads or writes the catalog record's tags. A resource holds at most 50 tags. Co-Authored-By: Claude Opus 5.5 * s3vectors: apply the tags passed to CreateVectorBucket and CreateIndex The tags field was accepted and dropped. It is now stored on the new bucket or index, and creating with tags needs s3vectors:TagResource in addition to the create action. Co-Authored-By: Claude Opus 5.5 * test/s3vectors: cover resource tagging end to end Co-Authored-By: Claude Opus 5.5 * s3tables: keep tag operations to a bound table bucket TagResource, UntagResource and ListTagsForResource run through Execute with WithTableBucketCreatedAt now answer not found when the bucket name holds another bucket, as the bound creates and deletes already do. Co-Authored-By: Claude Opus 5.5 * s3vectors: bind tagging to the authorized bucket and to vector indexes The catalog tag calls carry the creation time of the bucket authorizeVector approved, so a bucket created again under the name is not touched. An index ARN must name a Lance table: another table in the vector namespace of a format-less bucket is refused, as DeleteIndex does. Co-Authored-By: Claude Opus 5.5 * s3vectors: match a TagResource grant scoped to the bucket being created Co-Authored-By: Claude Opus 5.5 * s3tables: merge tag writes into the tags the record holds at the write TagResource and UntagResource merged into the tags they read before the compare-and-swap and wrote that map on every attempt, so a tag write that landed in between was undone. The merge now runs inside the mutation. WithMaxTags lets an in-process caller cap the merged set there. Co-Authored-By: Claude Opus 5.5 * s3vectors: check the 50-tag limit inside the catalog's tag write Co-Authored-By: Claude Opus 5.5 * s3tables: check a tag operation's bound bucket after its reads Co-Authored-By: Claude Opus 5.5 * s3vectors: validate each tag key UntagResource removes Co-Authored-By: Claude Opus 5.5 * s3vectors: validate untag keys one at a time UntagResource sized a map by the number of submitted tag keys before validating them, so a large body of repeated keys allocated far more than the request itself, before authorization ran. Co-Authored-By: Claude Opus 5.5 * s3vectors: authorize UntagResource before validating its keys Validating a near-32 MiB body of tag keys costs about 13x its JSON decode, so an unauthorized caller could make the gateway spend that work on a request it then refuses. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- test/s3vectors/s3vectors_test.py | 35 ++ weed/s3api/s3tables/handler_bind_tags_test.go | 96 +++++ weed/s3api/s3tables/handler_policy.go | 117 ++++-- weed/s3api/s3tables/manager.go | 8 +- weed/s3api/s3vectors/handlers.go | 26 +- weed/s3api/s3vectors/server.go | 21 +- weed/s3api/s3vectors/tagging.go | 191 ++++++++++ weed/s3api/s3vectors/tagging_test.go | 355 ++++++++++++++++++ weed/s3api/s3vectors/types.go | 25 +- 9 files changed, 835 insertions(+), 39 deletions(-) create mode 100644 weed/s3api/s3tables/handler_bind_tags_test.go create mode 100644 weed/s3api/s3vectors/tagging.go create mode 100644 weed/s3api/s3vectors/tagging_test.go diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index 51ebc4b5b4..05f83943ee 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -417,6 +417,40 @@ def test_index_arn_validation(vectors, index): assert vectors.get_index(**index)["index"]["indexArn"] == arn +def test_resource_tags(vectors, vector_clients, index): + reader = vector_clients["reader"] + bucket_arn = vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"])["vectorBucket"]["vectorBucketArn"] + index_arn = vectors.get_index(**index)["index"]["indexArn"] + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {} + vectors.tag_resource(resourceArn=bucket_arn, tags={"team": "search", "env": "dev"}) + vectors.tag_resource(resourceArn=index_arn, tags={"team": "ranking"}) + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search", "env": "dev"} + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"} + vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["env", "absent"]) + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"} + expect_error("ValidationException", vectors.tag_resource, resourceArn=bucket_arn, + tags={f"k{i}": "v" for i in range(50)}) + expect_error("NotFoundException", vectors.list_tags_for_resource, resourceArn=bucket_arn + "/index/missing") + expect_error("AccessDeniedException", reader.list_tags_for_resource, resourceArn=bucket_arn) + expect_error("AccessDeniedException", reader.tag_resource, resourceArn=index_arn, tags={"team": "other"}) + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"} + + +def test_create_with_tags(vectors): + bucket = "tagged-" + uuid.uuid4().hex[:16] + bucket_arn = vectors.create_vector_bucket(vectorBucketName=bucket, tags={"team": "search"})["vectorBucketArn"] + try: + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"} + index_arn = vectors.create_index(vectorBucketName=bucket, indexName=INDEX, dataType="float32", dimension=4, + distanceMetric="cosine", tags={"tier": "hot"})["indexArn"] + try: + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"tier": "hot"} + finally: + vectors.delete_index(vectorBucketName=bucket, indexName=INDEX) + finally: + vectors.delete_vector_bucket(vectorBucketName=bucket) + + def test_invalid_credentials(cluster): client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown", aws_secret_access_key="unknown", region_name="us-east-1") @@ -458,3 +492,4 @@ def test_engine_without_credentials(open_cluster): {"key": "b", "data": {"float32": [0, 1]}}]) hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"] assert [h["key"] for h in hits] == ["a"] + diff --git a/weed/s3api/s3tables/handler_bind_tags_test.go b/weed/s3api/s3tables/handler_bind_tags_test.go new file mode 100644 index 0000000000..cce363229e --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_tags_test.go @@ -0,0 +1,96 @@ +package s3tables + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Tag operations bound to a table bucket answer not found once its name +// holds another bucket, for the bucket and for a table in it, and leave the +// tags as they were. +func TestTagOperationsKeepToTheBoundBucket(t *testing.T) { + fs, m := startRenameManager(t) + client := NewManagerClient(fs.Client) + tableARN, err := BuildTableARN(DefaultRegion, DefaultAccountID, renameTestBucket, "ns", "t") + require.NoError(t, err) + var bucket GetTableBucketResponse + require.NoError(t, m.Execute(context.Background(), client, "GetTableBucket", + &GetTableBucketRequest{TableBucketARN: mustBucketARN(t)}, &bucket, "")) + bound := WithTableBucketCreatedAt(context.Background(), bucket.CreatedAt) + stale := WithTableBucketCreatedAt(context.Background(), bucket.CreatedAt.Add(-time.Hour)) + + for _, arn := range []string{mustBucketARN(t), tableARN} { + require.NoError(t, m.Execute(bound, client, "TagResource", + &TagResourceRequest{ResourceARN: arn, Tags: map[string]string{"k": "v"}}, nil, ""), arn) + for operation, req := range map[string]interface{}{ + "TagResource": &TagResourceRequest{ResourceARN: arn, Tags: map[string]string{"k": "other"}}, + "UntagResource": &UntagResourceRequest{ResourceARN: arn, TagKeys: []string{"k"}}, + "ListTagsForResource": &ListTagsForResourceRequest{ResourceARN: arn}, + } { + err := m.Execute(stale, client, operation, req, nil, "") + var apiErr *S3TablesError + require.ErrorAs(t, err, &apiErr, "%s on %s", operation, arn) + assert.Contains(t, []string{ErrCodeNoSuchBucket, ErrCodeNoSuchTable}, apiErr.Type, "%s on %s", operation, arn) + } + var tags ListTagsForResourceResponse + require.NoError(t, m.Execute(bound, client, "ListTagsForResource", + &ListTagsForResourceRequest{ResourceARN: arn}, &tags, ""), arn) + assert.Equal(t, map[string]string{"k": "v"}, tags.Tags, arn) + } +} + +// A tag write merges into the tags the record holds when it is written, so a +// tag write that lands between its read and its write is kept. +func TestTagWritesKeepAConcurrentTagWrite(t *testing.T) { + for _, tc := range []struct { + operation string + input func(t *testing.T) interface{} + want map[string]string + }{ + {"TagResource", func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: map[string]string{"k": "v"}} + }, map[string]string{"keep": "1", "other": "1", "k": "v"}}, + {"UntagResource", func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: mustBucketARN(t), TagKeys: []string{"keep"}} + }, map[string]string{"other": "1"}}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + extended[ExtendedKeyTags] = []byte(`{"keep":"1"}`) + fs.Put(TablesPath, renameTestBucket, extended) + fs.BeforeUpdate = func() { + current := extendedOf(t, fs, TablesPath, renameTestBucket) + current[ExtendedKeyTags] = []byte(`{"keep":"1","other":"1"}`) + fs.Put(TablesPath, renameTestBucket, current) + } + code, resp := runHandler(t, m, fs.Client, tc.operation, tc.input(t)) + require.Equal(t, http.StatusOK, code, resp.Message) + var got map[string]string + require.NoError(t, json.Unmarshal(extendedOf(t, fs, TablesPath, renameTestBucket)[ExtendedKeyTags], &got)) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestTagResourceKeepsToItsLimit(t *testing.T) { + fs, m := startEmptyBucket(t) + client := NewManagerClient(fs.Client) + ctx := WithMaxTags(context.Background(), 2) + tag := func(tags map[string]string) error { + return m.Execute(ctx, client, "TagResource", &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: tags}, nil, "") + } + require.NoError(t, tag(map[string]string{"a": "1", "b": "1"})) + require.NoError(t, tag(map[string]string{"a": "2"})) + err := tag(map[string]string{"c": "1"}) + var apiErr *S3TablesError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, ErrCodeInvalidRequest, apiErr.Type) + assert.JSONEq(t, `{"a":"2","b":"1"}`, string(extendedOf(t, fs, TablesPath, renameTestBucket)[ExtendedKeyTags])) +} diff --git a/weed/s3api/s3tables/handler_policy.go b/weed/s3api/s3tables/handler_policy.go index 8483900637..cf455df912 100644 --- a/weed/s3api/s3tables/handler_policy.go +++ b/weed/s3api/s3tables/handler_policy.go @@ -2,6 +2,7 @@ package s3tables import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -43,6 +44,65 @@ func (h *S3TablesHandler) extractResourceOwnerAndBucket( return ownerAccountID, bucketName, createdAt, nil } +type maxTagsKey struct{} + +var errTooManyTags = errors.New("too many tags") + +// WithMaxTags caps how many tags a TagResource run through Execute may leave +// on its resource. In-process only, like WithTableBucketCreatedAt. +func WithMaxTags(ctx context.Context, n int) context.Context { + return context.WithValue(ctx, maxTagsKey{}, n) +} + +// updateTags applies set and remove to raw, the tags the record holds at the +// write, so concurrent tag writes do not undo each other. +func updateTags(raw []byte, set map[string]string, remove []string) (map[string]string, error) { + tags := make(map[string]string) + if len(raw) > 0 { + if err := json.Unmarshal(raw, &tags); err != nil { + return nil, err + } + } + for k, v := range set { + tags[k] = v + } + for _, k := range remove { + delete(tags, k) + } + return tags, nil +} + +func writeTags(extended map[string][]byte, key string, tags map[string]string) error { + data, err := json.Marshal(tags) + if err != nil { + return err + } + extended[key] = data + return nil +} + +// requireBoundTableBucket answers not found when ctx binds the request to a +// table bucket other than the one now named bucketName. +func (h *S3TablesHandler) requireBoundTableBucket(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName string) error { + if _, bound := ctx.Value(tableBucketCreatedAtKey{}).(time.Time); !bound { + return nil + } + data, err := h.getExtendedAttribute(ctx, client, GetTableBucketPath(bucketName), ExtendedKeyMetadata) + if errors.Is(err, ErrAttributeNotFound) { + return filer_pb.ErrNotFound + } else if err != nil { + return err + } + var meta tableBucketMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return err + } + if otherTableBucket(ctx, meta.CreatedAt) { + return filer_pb.ErrNotFound + } + return nil +} + // handlePutTableBucketPolicy puts a policy on a table bucket func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error { @@ -751,6 +811,10 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -794,28 +858,24 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } - // Merge new tags - for k, v := range req.Tags { - existingTags[k] = v - } - - // Write merged tags - tagsBytes, err := json.Marshal(existingTags) - if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") - return fmt.Errorf("failed to marshal tags: %w", err) - } sameResource := requireTableCreatedAt if rType != ResourceTypeTable { sameResource = requireTableBucketCreatedAt } + maxTags, limited := r.Context().Value(maxTagsKey{}).(int) err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { if err := sameResource(extended, resourcePath, createdAt); err != nil { return err } - extended[extendedKey] = tagsBytes - return nil + tags, err := updateTags(extended[extendedKey], req.Tags, nil) + if err != nil { + return err + } + if limited && len(tags) > maxTags { + return errTooManyTags + } + return writeTags(extended, extendedKey, tags) }) }) @@ -828,6 +888,11 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque h.writeError(w, http.StatusNotFound, errorCode, "resource not found") return err } + if errors.Is(err, errTooManyTags) { + h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, + fmt.Sprintf("a resource can have at most %d tags", maxTags)) + return err + } if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "resource tags changed concurrently, retry the request") return err @@ -909,6 +974,10 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -1034,6 +1103,10 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -1076,17 +1149,6 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } - // Remove specified tags - for _, key := range req.TagKeys { - delete(tags, key) - } - - // Write updated tags - tagsBytes, err := json.Marshal(tags) - if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") - return fmt.Errorf("failed to marshal tags: %w", err) - } sameResource := requireTableCreatedAt if rType != ResourceTypeTable { sameResource = requireTableBucketCreatedAt @@ -1096,8 +1158,11 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req if err := sameResource(extended, resourcePath, createdAt); err != nil { return err } - extended[extendedKey] = tagsBytes - return nil + tags, err := updateTags(extended[extendedKey], nil, req.TagKeys) + if err != nil { + return err + } + return writeTags(extended, extendedKey, tags) }) }) diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index e3be966476..0572d74a36 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -80,10 +80,10 @@ func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext { type tableBucketCreatedAtKey struct{} // WithTableBucketCreatedAt binds a DeleteTableBucket, CreateNamespace, -// DeleteNamespace, CreateTable or DeleteTable run through Execute to the -// bucket created at createdAt, so a caller that authorized the bucket itself -// never writes into one created again under its name. In-process only: no -// wire request can carry it. +// DeleteNamespace, CreateTable, DeleteTable or tag operation run through +// Execute to the bucket created at createdAt, so a caller that authorized the +// bucket itself never writes into one created again under its name. +// In-process only: no wire request can carry it. func WithTableBucketCreatedAt(ctx context.Context, createdAt time.Time) context.Context { return context.WithValue(ctx, tableBucketCreatedAtKey{}, createdAt) } diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 9f9be68b51..83018dca6b 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -39,6 +39,9 @@ var handlers = map[string]func(*Server, http.ResponseWriter, *http.Request){ "ListVectors": (*Server).handleListVectors, "DeleteVectors": (*Server).handleDeleteVectors, "QueryVectors": (*Server).handleQueryVectors, + "TagResource": (*Server).handleTagResource, + "UntagResource": (*Server).handleUntagResource, + "ListTagsForResource": (*Server).handleListTagsForResource, } // lance table ids put indexes under a "vector" namespace inside their bucket. @@ -275,6 +278,10 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } // The catalog keeps the s3tables field name; GetVectorBucket renames it back. enc := req.EncryptionConfiguration if len(enc) != 0 { @@ -283,7 +290,12 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request // No bucket exists yet, so check against the handler account as the // catalog does for CreateTableBucket. principal := s.tables.PrincipalFor(r) - if !s3tables.CheckPermissionWithContext("s3vectors:CreateVectorBucket", principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, "")) { + allowed := func(action, bucket string) bool { + return s3tables.CheckPermissionWithContext(action, principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, bucket)) + } + // Tagging names the new bucket, so a grant scoped to it applies. + if !allowed("s3vectors:CreateVectorBucket", "") || + (len(req.Tags) > 0 && !allowed("s3vectors:TagResource", req.VectorBucketName)) { catalogErr(w, errVectorDenied) return } @@ -293,6 +305,7 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request Name: req.VectorBucketName, Format: s3tables.FormatLance, EncryptionConfiguration: enc, + Tags: req.Tags, }, &resp, identityOf(r)) if err != nil { catalogErr(w, err) @@ -963,6 +976,10 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "dimension must be 1..=4096") return } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } // The create grant is exercised before any write: a denied caller must not // leave even the implicit namespace behind. bucketRecord, err := s.authorizeVector(r.Context(), r, "s3vectors:CreateIndex", arns.bucket, arns.index) @@ -970,6 +987,12 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { catalogErr(w, err) return } + if len(req.Tags) > 0 { + if _, err := s.authorizeVector(r.Context(), r, "s3vectors:TagResource", arns.bucket, arns.index); err != nil { + catalogErr(w, err) + return + } + } // The catalog writes land only in the bucket authorized above. bound := s3tables.WithTableBucketCreatedAt(r.Context(), bucketRecord.CreatedAt) // The namespace is implicit on AWS and internal bookkeeping here. @@ -1014,6 +1037,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { Name: arns.index, Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadataJSON}, + Tags: req.Tags, }, nil) if err != nil { catalogErr(w, err) diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index c2bb46a607..b901881eaf 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -69,17 +69,24 @@ func (s *Server) SetAccountID(id string) { s.accountID = id } // apply the AES256 a bucket declares. func (s *Server) SetEncryptionChecker(fn func() bool) { s.internal.SetEncryptionChecker(fn) } -// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header and -// POST /{Operation} for clients that route per-path. Metrics wrap Auth like the -// S3 track middleware so rejected requests still count toward the operation. +// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header, +// POST /{Operation} for clients that route per-path, and the tagging calls on +// /tags/{resourceArn}. Metrics wrap Auth like the S3 track middleware so +// rejected requests still count toward the operation. func (s *Server) RegisterRoutes(router *mux.Router) { - router.HandleFunc("/", s.instrument(s.Auth(s.dispatch))).Methods(http.MethodPost) - router.HandleFunc("/{op}", s.instrument(s.Auth(s.dispatch))).Methods(http.MethodPost) + handler := s.instrument(s.Auth(s.dispatch)) + router.HandleFunc("/", handler).Methods(http.MethodPost) + router.HandleFunc(tagsPathPrefix+"{resourceArn:.+}", handler).Methods(http.MethodGet, http.MethodPost, http.MethodDelete) + router.HandleFunc("/{op}", handler).Methods(http.MethodPost) } -// operationFromRequest resolves the operation the way dispatch does: the last -// path segment, or the X-Amz-Target suffix when the path is bare. +// operationFromRequest resolves the operation the way dispatch does: the +// method on a /tags/ path, else the last path segment, or the X-Amz-Target +// suffix when the path is bare. func operationFromRequest(r *http.Request) string { + if strings.HasPrefix(r.URL.Path, tagsPathPrefix) { + return tagOperation(r.Method) + } op := strings.Trim(r.URL.Path, "/") if i := strings.LastIndex(op, "/"); i >= 0 { op = op[i+1:] diff --git a/weed/s3api/s3vectors/tagging.go b/weed/s3api/s3vectors/tagging.go new file mode 100644 index 0000000000..c7370a67f1 --- /dev/null +++ b/weed/s3api/s3vectors/tagging.go @@ -0,0 +1,191 @@ +package s3vectors + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" +) + +// maxResourceTags is how many tags a vector bucket or index may carry. +const maxResourceTags = 50 + +const tagsPathPrefix = "/tags/" + +// tagOperation names the tagging call a /tags/{resourceArn} request makes. +func tagOperation(method string) string { + switch method { + case http.MethodGet: + return "ListTagsForResource" + case http.MethodDelete: + return "UntagResource" + } + return "TagResource" +} + +// readTagRequest decodes the body of a POST; GET and DELETE carry everything +// in the path and query. +func readTagRequest(r *http.Request, req interface{}) error { + if r.Method != http.MethodPost { + return nil + } + return readJSON(r, req) +} + +// tagTarget resolves a tagging call's resourceArn, taken from the path when +// the request was routed there, to the vector bucket or index and the ARN the +// catalog keys that record by. +func (s *Server) tagTarget(r *http.Request, bodyArn string) (vectorBucketARNs, string, error) { + arn := bodyArn + if v := mux.Vars(r)["resourceArn"]; v != "" { + arn = v + } + var v vectorBucketARNs + if arn == "" { + return v, "", fmt.Errorf("resourceArn is required") + } + if !strings.HasPrefix(arn, "arn:") { + return v, "", fmt.Errorf("resourceArn must be an ARN, got %q", arn) + } + if err := s.arnParts(&v, "", arn); err != nil { + return v, "", err + } + catalogArn := s.s3tablesArn(v.bucket) + if v.index != "" { + catalogArn = fmt.Sprintf("%s/table/%s/%s", catalogArn, indexNamespace, v.index) + } + return v, catalogArn, nil +} + +// authorizeTagging grants action on the tagged bucket or index and returns a +// context binding the catalog calls to the bucket it authorized. An index +// target must be a vector index, not another table in its namespace. +func (s *Server) authorizeTagging(r *http.Request, action string, arns vectorBucketARNs) (context.Context, error) { + record, err := s.authorizeVector(r.Context(), r, action, arns.bucket, arns.index) + if err != nil { + return nil, err + } + ctx := s3tables.WithTableBucketCreatedAt(r.Context(), record.CreatedAt) + if arns.index != "" { + index, err := s.indexRecord(ctx, arns.bucket, arns.index) + if err != nil { + return nil, err + } + if format, _ := index["format"].(string); format != s3tables.FormatLance { + return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeInvalidRequest, + Message: fmt.Sprintf("index %q is not a vector index", arns.index)} + } + } + return ctx, nil +} + +func (s *Server) resourceTags(ctx context.Context, catalogArn string) (map[string]string, error) { + var resp s3tables.ListTagsForResourceResponse + if err := s.internalCatalog(ctx, "ListTagsForResource", + &s3tables.ListTagsForResourceRequest{ResourceARN: catalogArn}, &resp); err != nil { + return nil, err + } + if resp.Tags == nil { + resp.Tags = map[string]string{} + } + return resp.Tags, nil +} + +func (s *Server) handleTagResource(w http.ResponseWriter, r *http.Request) { + var req TagResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if len(req.Tags) == 0 { + writeError(w, http.StatusBadRequest, "ValidationException", "tags are required") + return + } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:TagResource", arns) + if err != nil { + catalogErr(w, err) + return + } + if err := s.internalCatalog(s3tables.WithMaxTags(ctx, maxResourceTags), "TagResource", + &s3tables.TagResourceRequest{ResourceARN: catalogArn, Tags: req.Tags}, nil); err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &TagResourceResponse{}) +} + +func (s *Server) handleUntagResource(w http.ResponseWriter, r *http.Request) { + var req UntagResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if keys, ok := r.URL.Query()["tagKeys"]; ok { + req.TagKeys = keys + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if len(req.TagKeys) == 0 { + writeError(w, http.StatusBadRequest, "ValidationException", "tagKeys are required") + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:UntagResource", arns) + if err != nil { + catalogErr(w, err) + return + } + key := make(map[string]string, 1) + for _, k := range req.TagKeys { + clear(key) + key[k] = "" + if err := s3tables.ValidateTagsWithLimit(key, 1); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + } + if err := s.internalCatalog(ctx, "UntagResource", + &s3tables.UntagResourceRequest{ResourceARN: catalogArn, TagKeys: req.TagKeys}, nil); err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &UntagResourceResponse{}) +} + +func (s *Server) handleListTagsForResource(w http.ResponseWriter, r *http.Request) { + var req ListTagsForResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:ListTagsForResource", arns) + if err != nil { + catalogErr(w, err) + return + } + tags, err := s.resourceTags(ctx, catalogArn) + if err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &ListTagsForResourceResponse{Tags: tags}) +} diff --git a/weed/s3api/s3vectors/tagging_test.go b/weed/s3api/s3vectors/tagging_test.go new file mode 100644 index 0000000000..d68eec4896 --- /dev/null +++ b/weed/s3api/s3vectors/tagging_test.go @@ -0,0 +1,355 @@ +package s3vectors + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +type tagClient struct { + t *testing.T + ts *httptest.Server + fs *s3tablestest.MemFiler +} + +func newTagClient(t *testing.T) (*Server, *tagClient) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), headerAuth{}, newFakeVectors(0)) + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + ts := httptest.NewServer(router) + t.Cleanup(ts.Close) + return s, &tagClient{t: t, ts: ts, fs: fs} +} + +func (c *tagClient) do(identity, method, path string, header http.Header, payload interface{}) (int, string) { + c.t.Helper() + var body io.Reader + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + c.t.Fatalf("marshal: %v", err) + } + body = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, c.ts.URL+path, body) + if err != nil { + c.t.Fatalf("build %s %s: %v", method, path, err) + } + for k, v := range header { + req.Header[k] = v + } + if identity != "" { + req.Header.Set("X-Test-Identity", identity) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(out) +} + +func (c *tagClient) op(identity, op string, payload interface{}) (int, string) { + return c.do(identity, http.MethodPost, "/"+op, nil, payload) +} + +func (c *tagClient) tag(identity, arn string, tags map[string]string) (int, string) { + return c.do(identity, http.MethodPost, tagsPathPrefix+url.PathEscape(arn), nil, map[string]interface{}{"tags": tags}) +} + +func (c *tagClient) untag(identity, arn string, keys ...string) (int, string) { + q := url.Values{"tagKeys": keys} + return c.do(identity, http.MethodDelete, tagsPathPrefix+url.PathEscape(arn)+"?"+q.Encode(), nil, nil) +} + +func (c *tagClient) list(identity, arn string) (int, map[string]string) { + c.t.Helper() + code, body := c.do(identity, http.MethodGet, tagsPathPrefix+url.PathEscape(arn), nil, nil) + if code != http.StatusOK { + return code, nil + } + var resp struct { + Tags map[string]string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil || resp.Tags == nil { + c.t.Fatalf("decode ListTagsForResource %q: %v", body, err) + } + return code, resp.Tags +} + +func mustStatus(t *testing.T, what string, code int, body string, want int) { + t.Helper() + if code != want { + t.Fatalf("%s = %d %s, want %d", what, code, body, want) + } +} + +func TestTagVectorBucketsAndIndexes(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + code, body = c.op("", "CreateIndex", &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, + }) + mustStatus(t, "create index", code, body, http.StatusOK) + bucketArn, indexArn := s.vectorBucketArn("vec-bucket"), s.indexArn("vec-bucket", "idx") + + if _, tags := c.list("", bucketArn); len(tags) != 0 { + t.Fatalf("new bucket tags = %v, want none", tags) + } + code, body = c.tag("", bucketArn, map[string]string{"team": "search", "env": "dev"}) + mustStatus(t, "tag bucket", code, body, http.StatusOK) + code, body = c.tag("", indexArn, map[string]string{"team": "ranking"}) + mustStatus(t, "tag index", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"env": "prod"}) + mustStatus(t, "retag bucket", code, body, http.StatusOK) + + if _, tags := c.list("", bucketArn); !reflect.DeepEqual(tags, map[string]string{"team": "search", "env": "prod"}) { + t.Fatalf("bucket tags = %v", tags) + } + if _, tags := c.list("", indexArn); !reflect.DeepEqual(tags, map[string]string{"team": "ranking"}) { + t.Fatalf("index tags = %v", tags) + } + + code, body = c.untag("", bucketArn, "env", "absent") + mustStatus(t, "untag bucket", code, body, http.StatusOK) + if _, tags := c.list("", bucketArn); !reflect.DeepEqual(tags, map[string]string{"team": "search"}) { + t.Fatalf("bucket tags after untag = %v", tags) + } + + // Clients that post the operation by name carry resourceArn in the body. + code, body = c.do("", http.MethodPost, "/", http.Header{"X-Amz-Target": {"AmazonS3Vectors.ListTagsForResource"}}, + &ListTagsForResourceRequest{ResourceArn: indexArn}) + mustStatus(t, "list by target header", code, body, http.StatusOK) + var resp ListTagsForResourceResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil || resp.Tags["team"] != "ranking" { + t.Fatalf("list by target header = %s (%v)", body, err) + } +} + +func TestTagResourceValidation(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + bucketArn := s.vectorBucketArn("vec-bucket") + + code, body = c.tag("", s.indexArn("vec-bucket", "missing"), map[string]string{"a": "b"}) + mustStatus(t, "tag missing index", code, body, http.StatusNotFound) + code, _ = c.list("", s.vectorBucketArn("no-such-bucket")) + mustStatus(t, "list missing bucket", code, "", http.StatusNotFound) + code, body = c.tag("", "vec-bucket", map[string]string{"a": "b"}) + mustStatus(t, "tag by name", code, body, http.StatusBadRequest) + code, body = c.tag("", "arn:aws:s3vectors:elsewhere:1:bucket/vec-bucket", map[string]string{"a": "b"}) + mustStatus(t, "tag foreign ARN", code, body, http.StatusBadRequest) + code, body = c.tag("", bucketArn, map[string]string{}) + mustStatus(t, "tag with no tags", code, body, http.StatusBadRequest) + code, body = c.tag("", bucketArn, map[string]string{"bad key!": "v"}) + mustStatus(t, "tag with bad key", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn) + mustStatus(t, "untag with no keys", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn, "") + mustStatus(t, "untag an empty key", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn, strings.Repeat("k", 129)) + mustStatus(t, "untag a key past 128 characters", code, body, http.StatusBadRequest) + + tags := map[string]string{} + for i := 0; i < maxResourceTags; i++ { + tags[fmt.Sprintf("k%d", i)] = "v" + } + code, body = c.tag("", bucketArn, tags) + mustStatus(t, "tag up to the limit", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"k0": "changed"}) + mustStatus(t, "overwrite at the limit", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"one-more": "v"}) + mustStatus(t, "tag past the limit", code, body, http.StatusBadRequest) + tags["one-more"] = "v" + code, body = c.tag("", s.vectorBucketArn("vec-bucket"), tags) + mustStatus(t, "request past the limit", code, body, http.StatusBadRequest) + if _, got := c.list("", bucketArn); len(got) != maxResourceTags || got["k0"] != "changed" { + t.Fatalf("tags after refused requests: %d tags, k0=%q", len(got), got["k0"]) + } +} + +func TestTaggingNeedsItsOwnGrant(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + bucketArn := s.vectorBucketArn("vec-bucket") + code, body = c.tag("", bucketArn, map[string]string{"team": "search"}) + mustStatus(t, "owner tags", code, body, http.StatusOK) + + code, _ = c.list("reader", bucketArn) + mustStatus(t, "list without a grant", code, "", http.StatusForbidden) + code, body = c.op("", "PutVectorBucketPolicy", &PutVectorBucketPolicyRequest{ + VectorBucketName: "vec-bucket", + Policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*",` + + `"Action":["s3vectors:ListTagsForResource","s3vectors:GetVectorBucket"],"Resource":"` + bucketArn + `"}]}`, + }) + mustStatus(t, "put policy", code, body, http.StatusOK) + if code, tags := c.list("reader", bucketArn); code != http.StatusOK || tags["team"] != "search" { + t.Fatalf("list under grant = %d %v", code, tags) + } + code, body = c.tag("reader", bucketArn, map[string]string{"team": "other"}) + mustStatus(t, "tag under list grant", code, body, http.StatusForbidden) + code, body = c.untag("reader", bucketArn, "team") + mustStatus(t, "untag under list grant", code, body, http.StatusForbidden) + code, body = c.untag("reader", bucketArn, "") + mustStatus(t, "untag an empty key under list grant", code, body, http.StatusForbidden) + if _, tags := c.list("", bucketArn); tags["team"] != "search" { + t.Fatalf("refused writes changed tags: %v", tags) + } +} + +func TestCreateWithTagsNeedsTagResource(t *testing.T) { + e := newAuthTestEnv(t) + tags := map[string]string{"team": "search"} + listed := func(arn string) map[string]string { + t.Helper() + w := e.vectorResp(adminIdentity(), e.s.handleListTagsForResource, &ListTagsForResourceRequest{ResourceArn: arn}) + var resp ListTagsForResourceResponse + if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &resp) != nil { + t.Fatalf("list tags of %s: %d %s", arn, w.Code, w.Body.String()) + } + return resp.Tags + } + creator := func(actions ...s3api.Action) *s3api.Identity { + return &s3api.Identity{Name: "creator", Account: &s3api.Account{Id: "creator"}, Actions: actions} + } + createOnly := creator("s3vectors:CreateVectorBucket", "s3vectors:CreateIndex") + withTag := creator("s3vectors:CreateVectorBucket", "s3vectors:CreateIndex", "s3vectors:TagResource") + + if code := e.vector(createOnly, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "denied", Tags: tags}); code != http.StatusForbidden { + t.Fatalf("tagged bucket create without TagResource: %d, want 403", code) + } + if code := e.vector(adminIdentity(), e.s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "denied"}); code != http.StatusNotFound { + t.Fatalf("refused create left a bucket: %d", code) + } + if code := e.vector(withTag, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket", Tags: tags}); code != http.StatusOK { + t.Fatalf("tagged bucket create: %d", code) + } + if got := listed(e.s.vectorBucketArn("vec-bucket")); !reflect.DeepEqual(got, tags) { + t.Fatalf("bucket tags = %v, want %v", got, tags) + } + + e.createBuckets(adminIdentity(), "shared") + index := func(ident *s3api.Identity, name string, tags map[string]string) int { + return e.vector(ident, e.s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "v-shared", IndexName: name, DataType: "float32", Dimension: 3, Tags: tags, + }) + } + if code := index(createOnly, "denied", tags); code != http.StatusForbidden { + t.Fatalf("tagged index create without TagResource: %d, want 403", code) + } + if code := index(createOnly, "plain", nil); code != http.StatusOK { + t.Fatalf("untagged index create: %d", code) + } + if code := index(withTag, "tagged", tags); code != http.StatusOK { + t.Fatalf("tagged index create: %d", code) + } + if got := listed(e.s.indexArn("v-shared", "tagged")); !reflect.DeepEqual(got, tags) { + t.Fatalf("index tags = %v, want %v", got, tags) + } + if got := listed(e.s.indexArn("v-shared", "plain")); len(got) != 0 { + t.Fatalf("untagged index tags = %v", got) + } + + scoped := creator("s3vectors:CreateVectorBucket", "s3vectors:TagResource:team-bucket") + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "other-bucket", Tags: tags}); code != http.StatusForbidden { + t.Fatalf("tagged create outside the scoped TagResource grant: %d, want 403", code) + } + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "team-bucket", Tags: tags}); code != http.StatusOK { + t.Fatalf("tagged create under a TagResource grant scoped to the bucket: %d", code) + } + + tooMany := map[string]string{} + for i := 0; i <= maxResourceTags; i++ { + tooMany[fmt.Sprintf("k%d", i)] = "v" + } + if code := e.vector(withTag, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "too-many", Tags: tooMany}); code != http.StatusBadRequest { + t.Fatalf("bucket create past the tag limit: %d, want 400", code) + } + if code := index(withTag, "too-many", tooMany); code != http.StatusBadRequest { + t.Fatalf("index create past the tag limit: %d, want 400", code) + } +} + +func TestTaggingRefusesTablesThatAreNotIndexes(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + code, body = c.op("", "CreateIndex", &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, + }) + mustStatus(t, "create index", code, body, http.StatusOK) + indexArn := s.indexArn("vec-bucket", "idx") + code, body = c.tag("", indexArn, map[string]string{"team": "search"}) + mustStatus(t, "tag index", code, body, http.StatusOK) + + dir, name := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx")) + entry := c.fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Fatalf("decode index metadata: %v", err) + } + meta["format"] = s3tables.FormatIceberg + data, _ := json.Marshal(meta) + entry.Extended[s3tables.ExtendedKeyMetadata] = data + c.fs.Put(dir, name, entry.Extended) + + code, _ = c.list("", indexArn) + mustStatus(t, "list tags of an Iceberg table", code, "", http.StatusBadRequest) + code, body = c.tag("", indexArn, map[string]string{"team": "other"}) + mustStatus(t, "tag an Iceberg table", code, body, http.StatusBadRequest) + code, body = c.untag("", indexArn, "team") + mustStatus(t, "untag an Iceberg table", code, body, http.StatusBadRequest) + if got := string(c.fs.Get(dir, name).Extended[s3tables.ExtendedKeyTags]); got != `{"team":"search"}` { + t.Fatalf("refused calls changed the table's tags: %s", got) + } +} + +func TestUntagManyKeysAllocatesLikeDecoding(t *testing.T) { + s, c := newTagClient(t) + code, out := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, out, http.StatusOK) + body, err := json.Marshal(&UntagResourceRequest{ResourceArn: s.vectorBucketArn("vec-bucket"), TagKeys: make([]string, 1<<18)}) + if err != nil { + t.Fatal(err) + } + allocated := func(f func()) uint64 { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + f() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc + } + decode := allocated(func() { + var req UntagResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Fatal(err) + } + }) + w := httptest.NewRecorder() + handle := allocated(func() { + s.handleUntagResource(w, httptest.NewRequest(http.MethodPost, "/UntagResource", bytes.NewReader(body))) + }) + mustStatus(t, "untag empty keys", w.Code, w.Body.String(), http.StatusBadRequest) + if handle > decode*3/2 { + t.Fatalf("untag allocated %d bytes, decoding the request takes %d", handle, decode) + } +} diff --git a/weed/s3api/s3vectors/types.go b/weed/s3api/s3vectors/types.go index 0bbbaff6f6..4f5f1496ec 100644 --- a/weed/s3api/s3vectors/types.go +++ b/weed/s3api/s3vectors/types.go @@ -62,7 +62,7 @@ type vectorBucketARNs struct { type CreateVectorBucketRequest struct { VectorBucketName string `json:"vectorBucketName"` EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"` - MetadataResourceTags map[string]string `json:"-"` + Tags map[string]string `json:"tags,omitempty"` } type CreateVectorBucketResponse struct { @@ -136,6 +136,7 @@ type CreateIndexRequest struct { MetadataConfiguration *MetadataConfiguration `json:"metadataConfiguration,omitempty"` VectorBucketName string `json:"vectorBucketName"` VectorBucketArn string `json:"vectorBucketArn"` + Tags map[string]string `json:"tags,omitempty"` } type MetadataConfiguration struct { @@ -295,3 +296,25 @@ type QueryVectorsResponse struct { DistanceMetric string `json:"distanceMetric"` Vectors []*QueryOutputVector `json:"vectors"` } + +type TagResourceRequest struct { + ResourceArn string `json:"resourceArn"` + Tags map[string]string `json:"tags"` +} + +type TagResourceResponse struct{} + +type UntagResourceRequest struct { + ResourceArn string `json:"resourceArn"` + TagKeys []string `json:"tagKeys"` +} + +type UntagResourceResponse struct{} + +type ListTagsForResourceRequest struct { + ResourceArn string `json:"resourceArn"` +} + +type ListTagsForResourceResponse struct { + Tags map[string]string `json:"tags"` +} From 0176231ea467021f6106dd58b3b2e0b06acc3338 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 11:24:51 -0800 Subject: [PATCH 29/31] volume: replicas keep the primary's compression and metadata (#12350) * volume(rust): replicate the stored needle instead of the request body The primary forwarded the original request body and headers, so a replica re-derived compression from the raw bytes. Send what Go's ReplicatedWrite sends: a one-part multipart upload of the stored bytes, gzip-marked when compressed, with the name, mime, pairs and Content-MD5, and carry ts, ttl, cm and fsync in the query so form-field values reach the replica too. Co-Authored-By: Claude Opus 5.5 * volume(rust): keep the primary's compression on replica writes A type=replicate upload now stores the bytes as received, like Go's ParseUpload, so a replica no longer compresses data the primary stored uncompressed. Co-Authored-By: Claude Opus 5.5 * volume(rust): replicas keep the primary's name and mime Forward the primary's filename and mime even past the 255-byte needle fields, so a replica applies the same limit and sets the same flags, and skip the filename MIME guess on type=replicate, so an untyped upload stays untyped on every copy. Co-Authored-By: Claude Opus 5.5 * volume(rust): keep the multipart part Content-Type as sent Parsing the part type as a mime value lowercased it and dropped values it could not parse, so a replica stored a different mime than the primary. Read the header verbatim, as Go's part.Header.Get does. Co-Authored-By: Claude Opus 5.5 * volume: replicas keep the primary's mime as forwarded A Go replica guessed the mime from the file extension, so an upload the primary stored as application/octet-stream, or with no mime at all, got a different type on the replica. On type=replicate keep the part's Content-Type as sent, and have the replication sender leave an empty mime empty instead of filling it in from the extension. This matches what the Rust replica does. Co-Authored-By: Claude Opus 5.5 * volume(rust): pick the JPEG orientation fix by file name, as Go does Go checks the mime and the upload's file name, Rust checked the mime and the URL path, so the same upload could be rotated on one server and not on the other. With the name and mime forwarded as stored, a replica now reaches the primary's decision, and a rotated image has no EXIF tag left to act on, so the fix stays a no-op there. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/images.rs | 6 +- seaweed-volume/src/server/handlers.rs | 429 +++++++++++++++++++-- weed/operation/needle_parse_test.go | 21 + weed/operation/upload_content.go | 3 +- weed/storage/needle/needle_parse_upload.go | 4 +- 5 files changed, 422 insertions(+), 41 deletions(-) diff --git a/seaweed-volume/src/images.rs b/seaweed-volume/src/images.rs index 9ad7ca71c8..2125f433da 100644 --- a/seaweed-volume/src/images.rs +++ b/seaweed-volume/src/images.rs @@ -143,12 +143,12 @@ fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage { DynamicImage::ImageRgba8(out) } -/// Returns true if the given MIME type or file path extension indicates a JPEG file. -pub fn is_jpeg(mime_type: &str, path: &str) -> bool { +/// Returns true if the given MIME type or file name extension indicates a JPEG file. +pub fn is_jpeg(mime_type: &str, filename: &str) -> bool { if mime_type == "image/jpeg" { return true; } - let lower = path.to_lowercase(); + let lower = filename.to_lowercase(); lower.ends_with(".jpg") || lower.ends_with(".jpeg") } diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 91c2d2a7ca..e4c150c5ea 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -695,6 +695,27 @@ async fn batch_delete_file_ids( Ok(()) } +/// `query` with the fields named in `overrides` replaced by their values. +fn override_query_fields(query: &str, overrides: &[(&str, String)]) -> Result { + let mut fields = Vec::new(); + for field in query.split('&').filter(|field| !field.is_empty()) { + let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field) + .map_err(|e| format!("parse replication query: {e}"))?; + if !decoded + .iter() + .any(|(name, _)| overrides.iter().any(|(o, _)| o == name)) + { + fields.push(field); + } + } + let encoded = serde_urlencoded::to_string(overrides) + .map_err(|e| format!("encode replication query: {e}"))?; + if !encoded.is_empty() { + fields.push(&encoded); + } + Ok(fields.join("&")) +} + /// Helper to synchronously replicate a request to peer volume servers. async fn do_replicated_request( state: &VolumeServerState, @@ -756,16 +777,7 @@ async fn do_replicated_request( return Ok(()); } - let mut query_fields = Vec::new(); - for field in query.split('&').filter(|field| !field.is_empty()) { - let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field) - .map_err(|e| format!("parse replication query: {e}"))?; - if !decoded.iter().any(|(name, _)| name == "type") { - query_fields.push(field); - } - } - query_fields.push("type=replicate"); - let new_query = query_fields.join("&"); + let new_query = override_query_fields(query, &[("type", "replicate".to_string())])?; let mut futures = Vec::new(); for loc in remote_locations { @@ -793,6 +805,12 @@ async fn do_replicated_request( if let Some(auth) = headers.get(axum::http::header::AUTHORIZATION) { req_builder = req_builder.header(axum::http::header::AUTHORIZATION, auth); } + for (name, value) in headers + .iter() + .filter(|(name, _)| name.as_str().starts_with("seaweed-")) + { + req_builder = req_builder.header(name, value); + } if let Some(ref b) = body { req_builder = req_builder.body(b.clone()); @@ -2855,8 +2873,12 @@ pub async fn post_handler( while let Ok(Some(field)) = multipart.next_field().await { let field_name = field.name().map(|s| s.to_string()); let fname = field.file_name().map(clean_windows_path_base); - let fct = field.content_type().map(|m| m.to_string()); let field_headers = field.headers().clone(); + let fct = field_headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); let fce = field_headers .get(header::CONTENT_ENCODING) .and_then(|v| v.to_str().ok()) @@ -3022,15 +3044,18 @@ pub async fn post_handler( let mime_type = if let Some(ref pct) = parsed_content_type { pct.clone() } else { - let multipart_fallback = - if should_parse_multipart && !filename.is_empty() && !is_chunk_manifest { - mime_guess::from_path(&filename) - .first() - .map(|m| m.to_string()) - .unwrap_or_default() - } else { - String::new() - }; + let multipart_fallback = if should_parse_multipart + && !filename.is_empty() + && !is_chunk_manifest + && !is_replicate + { + mime_guess::from_path(&filename) + .first() + .map(|m| m.to_string()) + .unwrap_or_default() + } else { + String::new() + }; headers .get(header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) @@ -3093,15 +3118,19 @@ pub async fn post_handler( .collect(); // Fix JPEG orientation from EXIF data before storing (matches Go behavior). - let body_data = if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &path) { - crate::images::fix_jpg_orientation(&body_data_raw) - } else { - body_data_raw - }; + let (body_data, body_rewritten) = + if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &filename) { + let fixed = crate::images::fix_jpg_orientation(&body_data_raw); + let rewritten = fixed != body_data_raw; + (fixed, rewritten) + } else { + (body_data_raw, false) + }; // Auto-compress compressible file types (matches Go's IsCompressableFileType). // Only compress if not already gzipped and compression saves >10%. - let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest { + // A replica keeps the compression state the primary stored. + let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest && !is_replicate { let ext = filename .rfind('.') .map(|p| &filename[p..]) @@ -3194,23 +3223,55 @@ pub async fn post_handler( }; // The fan-out runs concurrently with the local needle write, like Go's - // ReplicatedWrite: it reads the already-buffered request body, so the two - // sides share no mutable state. The ack still waits for both. + // ReplicatedWrite: replicas get the stored bytes with their compression + // state, so they keep it instead of re-deriving it. The ack waits for both. let replication = if needs_replication { let state_clone = state.clone(); let path_clone = path.clone(); let query_clone = query.clone(); - let headers_clone = headers.clone(); - let body_clone = body.clone(); + let mut overrides = vec![("ts", n.last_modified.to_string())]; + if let Some(t) = ttl.filter(|t| !t.is_empty()) { + overrides.push(("ttl", t.to_string())); + } + if is_chunk_manifest { + overrides.push(("cm", "true".to_string())); + } + if fsync { + overrides.push(("fsync", "true".to_string())); + } + let mut replica_headers = HeaderMap::new(); + if let Some(auth) = headers.get(header::AUTHORIZATION) { + replica_headers.insert(header::AUTHORIZATION, auth.clone()); + } + if n.has_pairs() { + for (key, value) in &pair_map { + if let (Ok(name), Ok(value)) = ( + header::HeaderName::try_from(format!("seaweed-{key}")), + header::HeaderValue::from_str(value), + ) { + replica_headers.insert(name, value); + } + } + } + let (content_type, replica_body) = replica_upload_body( + &filename, + &mime_type, + n.is_compressed().then_some("gzip"), + original_content_md5.as_deref().filter(|_| !body_rewritten), + &n.data, + ); + if let Ok(value) = header::HeaderValue::from_str(&content_type) { + replica_headers.insert(header::CONTENT_TYPE, value); + } Some(tokio::spawn(async move { do_replicated_request( &state_clone, vid.0, Method::POST, &path_clone, - &query_clone, - &headers_clone, - Some(body_clone), + &override_query_fields(&query_clone, &overrides)?, + &replica_headers, + Some(replica_body), ) .await })) @@ -4782,6 +4843,41 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { false } +/// One-part multipart upload of a stored needle, the form Go's upload_content +/// sends: the bytes as stored, labeled with their content encoding. +fn replica_upload_body( + filename: &str, + mime: &str, + content_encoding: Option<&str>, + content_md5: Option<&str>, + data: &[u8], +) -> (String, bytes::Bytes) { + let boundary = uuid::Uuid::new_v4().simple().to_string(); + let mut head = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n", + filename.replace('\\', "\\\\").replace('"', "\\\"") + ); + if !mime.is_empty() { + head.push_str(&format!("Content-Type: {mime}\r\n")); + } + if let Some(encoding) = content_encoding { + head.push_str(&format!("Content-Encoding: {encoding}\r\n")); + } + if let Some(md5) = content_md5 { + head.push_str(&format!("Content-MD5: {md5}\r\n")); + } + head.push_str("\r\n"); + let tail = format!("\r\n--{boundary}--\r\n"); + let mut body = Vec::with_capacity(head.len() + data.len() + tail.len()); + body.extend_from_slice(head.as_bytes()); + body.extend_from_slice(data); + body.extend_from_slice(tail.as_bytes()); + ( + format!("multipart/form-data; boundary={boundary}"), + body.into(), + ) +} + /// Try to gzip data. Returns None on error. fn try_gzip_data(data: &[u8]) -> Option> { use flate2::Compression; @@ -8228,12 +8324,15 @@ mod tests { ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); *state.current_master_url.write().await = listener.local_addr().unwrap().to_string(); - let master_server = tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); + let master_server = + tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); let id = 0x6e7a_0c05u64; let fid = format!("1,{id:x}{TEST_COOKIE:08x}"); let token = gen_jwt(&key, 60, &fid).unwrap(); - let query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&%74ype=client&type=old"; - let expected_query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&type=replicate"; + let query = + "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&%74ype=client&type=old"; + let expected_query = + "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&type=replicate"; let payload = b"authenticated replica bytes"; for method in [Method::POST, Method::DELETE] { let response = tokio::time::timeout( @@ -8282,6 +8381,264 @@ mod tests { peer_server.abort(); } + #[tokio::test] + async fn http_replica_stores_the_primary_needle() { + use crate::storage::super_block::ReplicaPlacement; + use axum::{Json, Router, routing::get}; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = volume_test_state(&tmp); + Arc::get_mut(&mut state).unwrap().self_url = "127.0.0.1:8080".to_string(); + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + volume + .set_replica_placement(ReplicaPlacement::from_string("001").unwrap()) + .unwrap(); + let peer_tmp = tempfile::TempDir::new().unwrap(); + let peer = volume_test_state(&peer_tmp); + let peer_volume = peer + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let peer_addr = listener.local_addr().unwrap(); + let peer_server = tokio::spawn(async move { + axum::serve( + listener, + super::super::volume_server::build_admin_router(peer), + ) + .await + .unwrap(); + }); + let master = Router::new().route( + "/dir/lookup", + get(move || async move { + Json(serde_json::json!({"locations": [ + {"url": "127.0.0.1:8080"}, + {"url": peer_addr.to_string()} + ]})) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + *state.current_master_url.write().await = listener.local_addr().unwrap().to_string(); + let master_server = + tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); + + let payload = "compressible replica text ".repeat(64); + let boundary = "replica-boundary"; + let multipart = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"ttl\"\r\n\r\n3m\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a \\\"b\\\".txt\"\r\n\ + Content-Type: text/plain\r\n\r\n{payload}\r\n--{boundary}--\r\n" + ); + let cases = [ + ( + 0x6e7a_0c06u64, + Request::builder() + .header(header::CONTENT_TYPE, "text/plain") + .header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"notes.txt\"", + ) + .header("Content-MD5", compute_md5_base64(payload.as_bytes())) + .header("Seaweed-Color", "blue"), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c07u64, + Request::builder().header( + header::CONTENT_TYPE, + format!("multipart/form-data; boundary={boundary}"), + ), + Body::from(multipart), + ), + ( + 0x6e7a_0c0au64, + Request::builder().header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"untyped.txt\"", + ), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c0bu64, + Request::builder().header( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{}.txt\"", "n".repeat(300)), + ), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c0eu64, + Request::builder() + .header(header::CONTENT_TYPE, "Text/Plain; charset=UTF-8") + .header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"upper.txt\"", + ), + Body::from(payload.clone()), + ), + ]; + for (id, request, body) in cases { + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + super::super::volume_server::build_admin_router(state.clone()).oneshot( + request + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}?ts=1700000000")) + .body(body) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let read = |v: &Arc| { + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + v.read_needle(&mut stored).unwrap(); + stored + }; + let (primary, replica) = (read(&volume), read(&peer_volume)); + assert!(primary.is_compressed()); + assert_eq!( + maybe_decompress_gzip(&primary.data).unwrap(), + payload.as_bytes() + ); + assert_eq!(replica.data, primary.data); + assert_eq!(replica.flags, primary.flags); + assert_eq!(replica.name, primary.name); + assert_eq!(replica.mime, primary.mime); + assert_eq!(replica.pairs, primary.pairs); + assert_eq!(replica.ttl, primary.ttl); + assert_eq!(replica.last_modified, 1700000000); + } + master_server.abort(); + peer_server.abort(); + } + + #[tokio::test] + async fn http_replica_keeps_uncompressed_bytes() { + use tower::ServiceExt; + let tmp = tempfile::TempDir::new().unwrap(); + let state = volume_test_state(&tmp); + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let payload = "compressible replica text ".repeat(64); + for (id, query, compressed) in [ + (0x6e7a_0c08u64, "?type=replicate", false), + (0x6e7a_0c09u64, "", true), + ] { + let response = super::super::volume_server::build_admin_router(state.clone()) + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}{query}")) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(payload.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + volume.read_needle(&mut stored).unwrap(); + assert_eq!(stored.is_compressed(), compressed, "query {query:?}"); + } + } + + #[tokio::test] + async fn http_orientation_fix_follows_the_filename() { + use tower::ServiceExt; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = volume_test_state(&tmp); + Arc::get_mut(&mut state).unwrap().fix_jpg_orientation = true; + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image::RgbaImage::new(3, 2)) + .write_to(&mut encoded, image::ImageFormat::Jpeg) + .unwrap(); + let exif: &[u8] = + b"Exif\0\0MM\0\x2a\0\0\0\x08\0\x01\x01\x12\0\x03\0\0\0\x01\0\x03\0\0\0\0\0\0"; + let mut tagged = vec![0xFF, 0xD8, 0xFF, 0xE1, 0, exif.len() as u8 + 2]; + tagged.extend_from_slice(exif); + tagged.extend_from_slice(&encoded.into_inner()[2..]); + let upload = |id: u64, suffix: &'static str, filename: &'static str, body: Vec| { + let (state, volume) = (state.clone(), volume.clone()); + async move { + let response = super::super::volume_server::build_admin_router(state) + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}{suffix}")) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{filename}\""), + ) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::CREATED, + "{suffix} {filename}" + ); + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + volume.read_needle(&mut stored).unwrap(); + stored.data + } + }; + let rotated = upload(0x6e7a_0c10, "", "photo.jpg", tagged.clone()).await; + assert_ne!(rotated, tagged); + assert_eq!( + upload(0x6e7a_0c11, ".jpg", "photo.png", tagged.clone()).await, + tagged + ); + assert_eq!( + upload(0x6e7a_0c12, "?type=replicate", "photo.jpg", tagged.clone()).await, + rotated + ); + assert_eq!( + upload(0x6e7a_0c13, "?type=replicate", "photo.jpg", rotated.clone()).await, + rotated + ); + } + #[tokio::test] async fn http_multipart_replica_refusal_is_not_marked() { use tower::ServiceExt; diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index fb1d530a78..6251ca771d 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -146,6 +146,27 @@ func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) { assert.Equal(t, data, stored.Data) } +func TestReplicationKeepsTheSourceMime(t *testing.T) { + for _, mimeType := range []string{"", "application/octet-stream", "Text/Plain; charset=UTF-8"} { + mockClient := &MockClient{} + var stored *needle.Needle + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { + assert.Equal(t, nil, err, "upload: %v", err) + stored = n + } + newUploader(mockClient).UploadData(context.Background(), []byte(textContent), &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0?type=replicate", + Filename: "report.html", + MimeType: mimeType, + IsReplication: true, + }) + if stored == nil { + t.Fatal("no needle reached the volume server") + } + assert.Equal(t, mimeType, string(stored.Mime)) + } +} + var textContent = `Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 4d829e33e7..71992dfe78 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -474,6 +474,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option Filename: option.Filename, Cipher: false, IsInputCompressed: contentIsGzipped, + IsReplication: option.IsReplication, MimeType: option.MimeType, PairMap: option.PairMap, Jwt: option.Jwt, @@ -510,7 +511,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction // properly handling non-ASCII characters and special characters h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename})) h.Set("Idempotency-Key", option.UploadUrl) - if option.MimeType == "" { + if option.MimeType == "" && !option.IsReplication { option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename))) } if option.MimeType != "" { diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 3a668de9b6..f0b07e21db 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -298,7 +298,9 @@ func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm")) - if !pu.IsChunkedFile { + if r.FormValue("type") == "replicate" { + pu.MimeType = contentType + } else if !pu.IsChunkedFile { dotIndex := strings.LastIndex(pu.FileName, ".") ext, mtype := "", "" From a074538c8f8331bf1f711909480f026c59f7645e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 11:25:24 -0800 Subject: [PATCH 30/31] admin: manage S3 vector buckets (#12334) * s3vectors: run an operation for trusted local tooling Like s3tables.Manager.Execute, Server.Execute runs one S3 Vectors operation through its handler as a given identity, without a signed request. The admin console can then manage vector buckets through the same validation, authorization and catalog steps as the API. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * admin: list, inspect, create and delete vector buckets over the API /api/s3vectors runs the S3 Vectors operations through Server.Execute as the admin account: list vector buckets with their encryption, read one bucket with its policy and each index's data type, dimension, distance metric and non-filterable keys, create and delete a bucket, and delete an index. Going through the operations keeps the API's rules, such as ICEBERG buckets not being vector buckets and the implicit index namespace not blocking a delete. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3vectors: export the rule for which table buckets are vector buckets IsVectorBucketFormat names the check ListVectorBuckets and isVectorBucket already make, so the admin console can tell the same buckets apart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * admin: vector bucket pages Object Store > Vector Buckets lists the vector buckets with their ARN, encryption and creation time, creates and deletes them, and opens one bucket with its indexes (data type, dimension, distance metric, non-filterable keys) and its policy, where an index can be deleted. A LANCE row on the Table Buckets page links to the same bucket's vector page, since both APIs serve it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * admin: stop the vector bucket and index lists at the list limit A page cut short by filtered ICEBERG buckets let the next page carry the list past s3TablesAdminListLimit. Co-Authored-By: Claude Opus 5.5 * admin: say when a vector bucket or index list is cut at the limit The lists stop at s3TablesAdminListLimit while more remain, so mark them truncated and point to an S3 Vectors client for the rest. Co-Authored-By: Claude Opus 5.5 * admin: indent the vector bucket policy A policy stored as one line ran off the right edge of the details page. Policies that are not valid JSON are shown as stored. Co-Authored-By: Claude Opus 5.5 * admin: show a deeply nested vector bucket policy as stored Indenting multiplies the size of a deeply nested document, so a policy past 32 levels of nesting is shown without indenting. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/admin/dash/admin_server.go | 4 + weed/admin/dash/s3vectors_management.go | 254 +++++++++ weed/admin/dash/s3vectors_management_test.go | 174 +++++++ weed/admin/handlers/admin_handlers.go | 50 ++ weed/admin/view/app/s3tables_buckets.templ | 4 + weed/admin/view/app/s3tables_buckets_templ.go | 290 ++++++----- .../view/app/s3vectors_bucket_details.templ | 210 ++++++++ .../app/s3vectors_bucket_details_templ.go | 392 ++++++++++++++ weed/admin/view/app/s3vectors_buckets.templ | 189 +++++++ .../admin/view/app/s3vectors_buckets_templ.go | 202 ++++++++ weed/admin/view/app/s3vectors_buckets_test.go | 80 +++ weed/admin/view/layout/layout.templ | 5 + weed/admin/view/layout/layout_templ.go | 485 +++++++++--------- weed/s3api/s3vectors/execute.go | 54 ++ weed/s3api/s3vectors/execute_test.go | 66 +++ weed/s3api/s3vectors/handlers.go | 14 +- 16 files changed, 2099 insertions(+), 374 deletions(-) create mode 100644 weed/admin/dash/s3vectors_management.go create mode 100644 weed/admin/dash/s3vectors_management_test.go create mode 100644 weed/admin/view/app/s3vectors_bucket_details.templ create mode 100644 weed/admin/view/app/s3vectors_bucket_details_templ.go create mode 100644 weed/admin/view/app/s3vectors_buckets.templ create mode 100644 weed/admin/view/app/s3vectors_buckets_templ.go create mode 100644 weed/admin/view/app/s3vectors_buckets_test.go create mode 100644 weed/s3api/s3vectors/execute.go create mode 100644 weed/s3api/s3vectors/execute_test.go diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 9c3eb6696f..eadd89b3c9 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -43,6 +43,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" "github.com/seaweedfs/seaweedfs/weed/worker/tasks" _ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store @@ -167,6 +168,7 @@ type AdminServer struct { collectionStatsCacheThreshold time.Duration s3TablesManager *s3tables.Manager + s3VectorsServer *s3vectors.Server icebergPort int lancePort int @@ -220,6 +222,8 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste bgCancel: bgCancel, } + server.s3VectorsServer = newS3VectorsServer(server) + // Initialize topic retention purger server.topicRetentionPurger = NewTopicRetentionPurger(server) diff --git a/weed/admin/dash/s3vectors_management.go b/weed/admin/dash/s3vectors_management.go new file mode 100644 index 0000000000..58204fd2bb --- /dev/null +++ b/weed/admin/dash/s3vectors_management.go @@ -0,0 +1,254 @@ +package dash + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" +) + +type S3VectorsBucketsData struct { + Username string `json:"username"` + Buckets []S3VectorsBucketSummary `json:"buckets"` + TotalBuckets int `json:"total_buckets"` + Truncated bool `json:"truncated,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} + +type S3VectorsBucketSummary struct { + Name string `json:"name"` + ARN string `json:"arn"` + CreatedAt time.Time `json:"createdAt"` + // Encryption is the bucket's sseType, or empty when it declares none. + Encryption string `json:"encryption,omitempty"` +} + +type S3VectorsBucketData struct { + Username string `json:"username"` + Bucket S3VectorsBucketSummary `json:"bucket"` + Policy string `json:"policy,omitempty"` + Indexes []*s3vectors.Index `json:"indexes"` + TotalIndexes int `json:"total_indexes"` + Truncated bool `json:"truncated,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} + +// The S3 Vectors operations reach the filer through the admin server's own +// filer connection. +type s3VectorsFilerClient struct{ s *AdminServer } + +func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error { + return c.s.WithFilerClient(fn) +} + +// newS3VectorsServer serves the catalog side of S3 Vectors for the admin +// console. It has no vector engine, so it never runs an operation that needs +// one, such as CreateIndex. +func newS3VectorsServer(s *AdminServer) *s3vectors.Server { + server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil) + server.SetRegion(s3tables.DefaultRegion) + server.SetAccountID(s3tables.DefaultAccountID) + return server +} + +func (s *AdminServer) executeS3VectorsOperation(ctx context.Context, operation string, req, resp interface{}) error { + return s.s3VectorsServer.Execute(ctx, operation, req, resp, s3_constants.AccountAdminId) +} + +const s3VectorsAdminPage = 500 + +func (s *AdminServer) s3VectorsBucketSummary(ctx context.Context, name string) (S3VectorsBucketSummary, error) { + var resp s3vectors.GetVectorBucketResponse + if err := s.executeS3VectorsOperation(ctx, "GetVectorBucket", &s3vectors.GetVectorBucketRequest{VectorBucketName: name}, &resp); err != nil { + return S3VectorsBucketSummary{}, err + } + b := resp.VectorBucket + summary := S3VectorsBucketSummary{Name: b.VectorBucketName, ARN: b.VectorBucketArn, CreatedAt: b.CreationTime} + if sse, ok := b.EncryptionConfiguration["sseType"].(string); ok { + summary.Encryption = sse + } + return summary, nil +} + +func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBucketsData, error) { + data := S3VectorsBucketsData{Buckets: []S3VectorsBucketSummary{}} + page := s3VectorsAdminPage + req := &s3vectors.ListVectorBucketsRequest{MaxResults: &page} + for { + var resp s3vectors.ListVectorBucketsResponse + if err := s.executeS3VectorsOperation(ctx, "ListVectorBuckets", req, &resp); err != nil { + return S3VectorsBucketsData{}, err + } + for _, b := range resp.VectorBuckets { + if len(data.Buckets) == s3TablesAdminListLimit { + data.Truncated = true + break + } + summary, err := s.s3VectorsBucketSummary(ctx, b.VectorBucketName) + if isS3VectorsNotFound(err) { + continue + } + if err != nil { + return S3VectorsBucketsData{}, err + } + data.Buckets = append(data.Buckets, summary) + } + if data.Truncated || resp.NextToken == "" { + break + } + req.NextToken = resp.NextToken + } + data.TotalBuckets = len(data.Buckets) + data.LastUpdated = time.Now() + return data, nil +} + +func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string) (S3VectorsBucketData, error) { + summary, err := s.s3VectorsBucketSummary(ctx, bucket) + if err != nil { + return S3VectorsBucketData{}, err + } + data := S3VectorsBucketData{Bucket: summary, Indexes: []*s3vectors.Index{}, LastUpdated: time.Now()} + var policy s3vectors.GetVectorBucketPolicyResponse + switch err := s.executeS3VectorsOperation(ctx, "GetVectorBucketPolicy", &s3vectors.GetVectorBucketPolicyRequest{VectorBucketName: bucket}, &policy); { + case err == nil: + data.Policy = policy.Policy + case !isS3VectorsNotFound(err): + return S3VectorsBucketData{}, err + } + page := s3VectorsAdminPage + req := &s3vectors.ListIndexesRequest{VectorBucketName: bucket, MaxResults: &page} + for { + var resp s3vectors.ListIndexesResponse + if err := s.executeS3VectorsOperation(ctx, "ListIndexes", req, &resp); err != nil { + return S3VectorsBucketData{}, err + } + for _, summary := range resp.Indexes { + if len(data.Indexes) == s3TablesAdminListLimit { + data.Truncated = true + break + } + var index s3vectors.GetIndexResponse + err := s.executeS3VectorsOperation(ctx, "GetIndex", &s3vectors.GetIndexRequest{VectorBucketName: bucket, IndexName: summary.IndexName}, &index) + if isS3VectorsNotFound(err) { + continue + } + if err != nil { + return S3VectorsBucketData{}, err + } + data.Indexes = append(data.Indexes, index.Index) + } + if data.Truncated || resp.NextToken == "" { + break + } + req.NextToken = resp.NextToken + } + data.TotalIndexes = len(data.Indexes) + return data, nil +} + +func (s *AdminServer) ListS3VectorsBucketsAPI(w http.ResponseWriter, r *http.Request) { + data, err := s.GetS3VectorsBucketsData(r.Context()) + if err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, data) +} + +func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + var req struct { + Name string `json:"name"` + } + if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { + writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) + return + } + var resp s3vectors.CreateVectorBucketResponse + if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusCreated, map[string]interface{}{"arn": resp.VectorBucketArn}) +} + +func (s *AdminServer) DeleteS3VectorsBucket(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + bucket := r.URL.Query().Get("bucket") + if bucket == "" { + writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required") + return + } + if err := s.executeS3VectorsOperation(r.Context(), "DeleteVectorBucket", &s3vectors.DeleteVectorBucketRequest{VectorBucketName: bucket}, nil); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Vector bucket deleted"}) +} + +func (s *AdminServer) GetS3VectorsBucketAPI(w http.ResponseWriter, r *http.Request) { + bucket := r.URL.Query().Get("bucket") + if bucket == "" { + writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required") + return + } + data, err := s.GetS3VectorsBucketData(r.Context(), bucket) + if err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, data) +} + +func (s *AdminServer) DeleteS3VectorsIndex(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + bucket, index := r.URL.Query().Get("bucket"), r.URL.Query().Get("index") + if bucket == "" || index == "" { + writeJSONError(w, http.StatusBadRequest, "bucket and index query parameters are required") + return + } + if err := s.executeS3VectorsOperation(r.Context(), "DeleteIndex", &s3vectors.DeleteIndexRequest{VectorBucketName: bucket, IndexName: index}, nil); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Index deleted"}) +} + +func isS3VectorsNotFound(err error) bool { + var e *s3vectors.Error + return errors.As(err, &e) && e.Code == "NotFoundException" +} + +func writeS3VectorsError(w http.ResponseWriter, err error) { + var e *s3vectors.Error + if !errors.As(err, &e) { + writeJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + status := http.StatusInternalServerError + switch e.Code { + case "ValidationException": + status = http.StatusBadRequest + case "AccessDeniedException": + status = http.StatusForbidden + case "NotFoundException": + status = http.StatusNotFound + case "ConflictException": + status = http.StatusConflict + case "ServiceUnavailableException": + status = http.StatusServiceUnavailable + } + writeJSONError(w, status, e.Message) +} diff --git a/weed/admin/dash/s3vectors_management_test.go b/weed/admin/dash/s3vectors_management_test.go new file mode 100644 index 0000000000..bd0c36a65d --- /dev/null +++ b/weed/admin/dash/s3vectors_management_test.go @@ -0,0 +1,174 @@ +package dash + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" +) + +func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient) { + t.Helper() + fs := s3tablestest.Start(t) + client := s3tables.NewManagerClient(fs.Client) + server := s3vectors.NewServer(client, nil, nil) + server.SetRegion(s3tables.DefaultRegion) + server.SetAccountID(s3tables.DefaultAccountID) + return &AdminServer{s3VectorsServer: server}, client +} + +// createS3VectorsIndexRecord writes the catalog record CreateIndex leaves +// behind; the admin server has no engine to run CreateIndex itself. +func createS3VectorsIndexRecord(t *testing.T, client *s3tables.ManagerClient, bucket, index string, metadata map[string]string) { + t.Helper() + m := s3tables.NewManager() + m.SetTrusted(true) + arn := "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket + ctx := context.Background() + err := m.Execute(ctx, client, "CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}, nil, s3_constants.AccountAdminId) + if err != nil && !strings.Contains(err.Error(), "already exists") { + t.Fatalf("create namespace: %v", err) + } + full, _ := json.Marshal(metadata) + if err := m.Execute(ctx, client, "CreateTable", &s3tables.CreateTableRequest{ + TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, Name: index, + Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: full}, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create index record: %v", err) + } +} + +func s3VectorsRequest(t *testing.T, handler http.HandlerFunc, method, target, body string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + handler(w, httptest.NewRequest(method, target, strings.NewReader(body))) + return w +} + +func TestS3VectorsBucketsListsOnlyVectorBuckets(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"media"}`); w.Code != http.StatusCreated { + t.Fatalf("create: %d %s", w.Code, w.Body) + } + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{ + VectorBucketName: "sealed", EncryptionConfiguration: map[string]interface{}{"sseType": "AES256"}, + }, nil); err != nil { + t.Fatalf("create sealed: %v", err) + } + if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ + Name: "warehouse", Format: s3tables.FormatIceberg, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create iceberg bucket: %v", err) + } + + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, b := range data.Buckets { + got[b.Name] = b.Encryption + if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/" + b.Name; b.ARN != want || b.CreatedAt.IsZero() { + t.Errorf("%s: arn %q created %v, want %q and a time", b.Name, b.ARN, b.CreatedAt, want) + } + } + if len(got) != 2 || got["media"] != "" || got["sealed"] != "AES256" || data.TotalBuckets != 2 || data.Truncated { + t.Fatalf("buckets %v (total %d), want media and sealed (AES256) without the iceberg bucket", got, data.TotalBuckets) + } +} + +func TestS3VectorsBucketShowsIndexesAndPolicy(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil { + t.Fatal(err) + } + createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{ + "dimension": "3", "distanceMetric": "cosine", "nonFilterableMetadataKeys": `["summary"]`, + }) + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:QueryVectors","Resource":"*"}]}` + if err := s.executeS3VectorsOperation(ctx, "PutVectorBucketPolicy", &s3vectors.PutVectorBucketPolicyRequest{VectorBucketName: "media", Policy: policy}, nil); err != nil { + t.Fatal(err) + } + + w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=media", "") + if w.Code != http.StatusOK { + t.Fatalf("get: %d %s", w.Code, w.Body) + } + var data S3VectorsBucketData + if err := json.Unmarshal(w.Body.Bytes(), &data); err != nil { + t.Fatal(err) + } + if data.Policy != policy { + t.Errorf("policy %q, want %q", data.Policy, policy) + } + if data.TotalIndexes != 1 { + t.Fatalf("indexes %+v", data.Indexes) + } + idx := data.Indexes[0] + if idx.IndexName != "movies" || idx.Dimension != 3 || idx.DistanceMetric != "cosine" || idx.DataType != "float32" || + idx.MetadataConfiguration == nil || len(idx.MetadataConfiguration.NonFilterableMetadataKeys) != 1 { + t.Fatalf("index %+v", idx) + } + + if w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=missing", ""); w.Code != http.StatusNotFound { + t.Fatalf("missing bucket: %d, want 404", w.Code) + } +} + +func TestS3VectorsDeleteIndexThenBucket(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil { + t.Fatal(err) + } + createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{"dimension": "3", "distanceMetric": "cosine"}) + + if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusConflict { + t.Fatalf("delete a bucket with an index: %d %s, want 409", w.Code, w.Body) + } + if w := s3VectorsRequest(t, s.DeleteS3VectorsIndex, http.MethodDelete, "/api/s3vectors/indexes?bucket=media&index=movies", ""); w.Code != http.StatusOK { + t.Fatalf("delete index: %d %s", w.Code, w.Body) + } + // The emptied index namespace does not keep the bucket alive. + if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusOK { + t.Fatalf("delete bucket: %d %s", w.Code, w.Body) + } + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil || data.TotalBuckets != 0 { + t.Fatalf("after delete: %+v, %v", data.Buckets, err) + } +} + +func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + // The ICEBERG bucket leaves the first page short, so the last page + // would carry the list past the limit. + if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ + Name: "iceberg", Format: s3tables.FormatIceberg, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatal(err) + } + for i := 0; i <= s3TablesAdminListLimit; i++ { + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: fmt.Sprintf("media-%04d", i)}, nil); err != nil { + t.Fatal(err) + } + } + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil { + t.Fatal(err) + } + if data.TotalBuckets != s3TablesAdminListLimit || !data.Truncated { + t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit) + } +} diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 007d1e14b1..80018e1d76 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -1,6 +1,7 @@ package handlers import ( + "errors" "net/http" "net/url" "strconv" @@ -14,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" "github.com/seaweedfs/seaweedfs/weed/stats" ) @@ -111,6 +113,8 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) { r.HandleFunc("/object-store/groups", h.groupHandlers.ShowGroups).Methods(http.MethodGet) r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet) + r.HandleFunc("/object-store/vector-buckets", h.ShowS3VectorsBuckets).Methods(http.MethodGet) + r.HandleFunc("/object-store/vector-buckets/{bucket}", h.ShowS3VectorsBucketDetails).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables", h.ShowS3TablesTables).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables/{table}", h.ShowS3TablesTableDetails).Methods(http.MethodGet) @@ -251,6 +255,13 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { s3TablesApi.HandleFunc("/tags", h.adminServer.ListS3TablesTags).Methods(http.MethodGet) s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete) + s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter() + s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet) + s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost) + s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete) + s3VectorsApi.HandleFunc("/indexes", h.adminServer.GetS3VectorsBucketAPI).Methods(http.MethodGet) + s3VectorsApi.Handle("/indexes", wrapWrite(h.adminServer.DeleteS3VectorsIndex)).Methods(http.MethodDelete) + filesApi := api.PathPrefix("/files").Subrouter() filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete) filesApi.Handle("/delete-multiple", wrapWrite(h.fileBrowserHandlers.DeleteMultipleFiles)).Methods(http.MethodDelete) @@ -373,6 +384,45 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque } } +// ShowS3VectorsBuckets renders the vector buckets page +func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) { + username := h.getUsername(r) + data, err := h.adminServer.GetS3VectorsBucketsData(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to get vector buckets: "+err.Error()) + return + } + data.Username = username + + w.Header().Set("Content-Type", "text/html") + viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context())) + if err := layout.Layout(viewCtx, app.S3VectorsBuckets(data)).Render(r.Context(), w); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error()) + } +} + +// ShowS3VectorsBucketDetails renders one vector bucket with its indexes and policy +func (h *AdminHandlers) ShowS3VectorsBucketDetails(w http.ResponseWriter, r *http.Request) { + username := h.getUsername(r) + data, err := h.adminServer.GetS3VectorsBucketData(r.Context(), mux.Vars(r)["bucket"]) + if err != nil { + status := http.StatusInternalServerError + var e *s3vectors.Error + if errors.As(err, &e) && e.Code == "NotFoundException" { + status = http.StatusNotFound + } + writeJSONError(w, status, "Failed to get vector bucket: "+err.Error()) + return + } + data.Username = username + + w.Header().Set("Content-Type", "text/html") + viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context())) + if err := layout.Layout(viewCtx, app.S3VectorsBucketDetails(data)).Render(r.Context(), w); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error()) + } +} + // ShowS3TablesNamespaces renders namespaces for a table bucket func (h *AdminHandlers) ShowS3TablesNamespaces(w http.ResponseWriter, r *http.Request) { username := h.getUsername(r) diff --git a/weed/admin/view/app/s3tables_buckets.templ b/weed/admin/view/app/s3tables_buckets.templ index 6533e32545..04ee616219 100644 --- a/weed/admin/view/app/s3tables_buckets.templ +++ b/weed/admin/view/app/s3tables_buckets.templ @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" ) templ S3TablesBuckets(data dash.S3TablesBucketsData) { @@ -160,6 +161,9 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) { { bucket.Name } { formatLabel(bucket.Format) } + if s3vectors.IsVectorBucketFormat(bucket.Format) { + vector bucket + } { bucket.OwnerAccountID } { bucket.ARN } diff --git a/weed/admin/view/app/s3tables_buckets_templ.go b/weed/admin/view/app/s3tables_buckets_templ.go index f469bf8481..1829d8083a 100644 --- a/weed/admin/view/app/s3tables_buckets_templ.go +++ b/weed/admin/view/app/s3tables_buckets_templ.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" ) func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { @@ -46,7 +47,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/config", data.IcebergPort))).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 119} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 25, Col: 119} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var2) if templ_7745c5c3_Err != nil { @@ -65,7 +66,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/table", data.LancePort))).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 114} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 32, Col: 114} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3) if templ_7745c5c3_Err != nil { @@ -83,7 +84,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 91} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 91} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4) if templ_7745c5c3_Err != nil { @@ -96,7 +97,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 145} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 145} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var5) if templ_7745c5c3_Err != nil { @@ -119,7 +120,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 105} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 105} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var6) if templ_7745c5c3_Err != nil { @@ -132,7 +133,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.IcebergPort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 162} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 162} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { @@ -151,7 +152,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 103} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 103} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var8) if templ_7745c5c3_Err != nil { @@ -164,7 +165,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var9 string templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.LancePort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 158} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 158} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { @@ -193,7 +194,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var10 string templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 81, Col: 47} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 82, Col: 47} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { @@ -206,7 +207,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var11 string templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 100, Col: 54} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 101, Col: 54} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) if templ_7745c5c3_Err != nil { @@ -220,7 +221,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var12 string templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 120, Col: 47} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 121, Col: 47} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { @@ -244,7 +245,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var13 string templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 160, Col: 28} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 161, Col: 28} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) if templ_7745c5c3_Err != nil { @@ -283,7 +284,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var16 string templ_7745c5c3_Var16, templ_7745c5c3_Err = templruntime.NewAttributeExpression(formatBadgeTitle(bucket.Format)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 99} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 99} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16) if templ_7745c5c3_Err != nil { @@ -310,220 +311,243 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var17 string templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(formatLabel(bucket.Format)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 130} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 130} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var18 string - templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 38} + if s3vectors.IsVectorBucketFormat(bucket.Format) { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "vector bucket") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) + templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 52} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 168, Col: 38} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var20 string - templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name)) + templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 82} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 52} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var21 string - templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 60} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 82} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var22 string + templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 60} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.PolicyStatementCount > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "Not configured") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "Not configured") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) if parseErr == nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\"> ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\" data-bucket-name=\"") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var29 string + templ_7745c5c3_Var29, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 161} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "\" title=\"Delete\">
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Buckets) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "
No table buckets found

Create your first S3 Tables bucket to get started.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "
No table buckets found

Create your first S3 Tables bucket to get started.

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 || data.LancePort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "
Client Examples
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
Client Examples
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "
ICEBERGDuckDB
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
ICEBERGDuckDB
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var29 string
-				templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
+				var templ_7745c5c3_Var30 string
+				templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
 LOAD iceberg;
 
 CREATE SECRET (
@@ -534,18 +558,18 @@ CREATE SECRET (
 
 SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 251, Col: 74}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 255, Col: 74}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "
ICEBERGPython (PyIceberg)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
ICEBERGPython (PyIceberg)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var30 string
-				templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
+				var templ_7745c5c3_Var31 string
+				templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
 
 catalog = load_catalog(
     name="seaweedfs",
@@ -558,13 +582,13 @@ catalog = load_catalog(
 
 namespaces = catalog.list_namespaces()`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 268, Col: 39}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 272, Col: 39}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -574,25 +598,25 @@ namespaces = catalog.list_namespaces()`) templ_7745c5c3_ElementWriter := templ_7745c5c3_Buffer templ_7745c5c3_Buffer := templ.GetBuffer() var templ_7745c5c3_ElementScripts []templ.ComponentScript - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
0)} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var31...) + var templ_7745c5c3_Var32 = []any{templ.KV("mt-4", data.IcebergPort > 0)} + templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var32...) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var32 string - templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var31).String()).Resolve(ctx) + var templ_7745c5c3_Var33 string + templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var32).String()).Resolve(ctx) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 1, Col: 0} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "\"") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -606,30 +630,30 @@ namespaces = catalog.list_namespaces()`) } templ.ReleaseBuffer(templ_7745c5c3_Buffer) } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, ">LANCEPython (lance-namespace)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, ">LANCEPython (lance-namespace)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var33 string
-				templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
+				var templ_7745c5c3_Var34 string
+				templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
 
 ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
 
 ns.list_namespaces(id=["my-table-bucket"])
 ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 281, Col: 70}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 285, Col: 70}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "
LANCEPython (pylance)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
LANCEPython (pylance)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var34 string
-				templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
+				var templ_7745c5c3_Var35 string
+				templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
 from lance_namespace import connect
 
 ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
@@ -638,84 +662,84 @@ table = ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])
 # The namespace vends both the location and the credentials to read it.
 dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 294, Col: 79}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 298, Col: 79}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
Create Table Bucket
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
Create Table Bucket
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.LancePort > 0 && data.IcebergPort <= 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else if data.LancePort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "
Lance Served over the Lance Namespace API. pylance, LanceDB, Ray.
A bucket holds one format. Tables of the other are refused.
The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.
Optional tags in key=value format.
Delete Table Bucket

Are you sure you want to delete the table bucket ?

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\" data-lance-port=\"") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var37 string + templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 169} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\">A bucket holds one format. Tables of the other are refused.
The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.
Optional tags in key=value format.
Delete Table Bucket

Are you sure you want to delete the table bucket ?

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -723,7 +747,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "
Table Bucket Policy
Provide a policy JSON; use Delete Policy to remove the policy.
Resource Tags
Loading...
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "
Table Bucket Policy
Provide a policy JSON; use Delete Policy to remove the policy.
Resource Tags
Loading...
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/s3vectors_bucket_details.templ b/weed/admin/view/app/s3vectors_bucket_details.templ new file mode 100644 index 0000000000..8a6bf1c21d --- /dev/null +++ b/weed/admin/view/app/s3vectors_bucket_details.templ @@ -0,0 +1,210 @@ +package app + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) { +
+
+

+ { data.Bucket.Name } +

+ +
+
+
+
+
+
ARN
+
{ data.Bucket.ARN }
+
Created
+
{ data.Bucket.CreatedAt.Format("2006-01-02 15:04:05") }
+
Encryption
+
+ if data.Bucket.Encryption != "" { + { data.Bucket.Encryption } + } else { + None + } +
+
+
+
+
+
+
+ { fmt.Sprintf("%d indexes", data.TotalIndexes) } + if data.Truncated { + { fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes) } + } +
+
+
+
+ + + + + + + + + + + + + + for _, index := range data.Indexes { + + + + + + + + + + } + if len(data.Indexes) == 0 { + + + + } + +
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
+ { index.IndexName } +
{ index.IndexArn }
+ 		{ index.DataType }{ fmt.Sprintf("%d", index.Dimension) }{ index.DistanceMetric } + if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 { + { strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ") } + } else { + None + } + { index.CreationTime.Format("2006-01-02 15:04") } + +
+ No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index. +
+
+
+
+
+
+
+ Bucket Policy +
+
+
+ if data.Policy != "" { + 
{ indentJSON(data.Policy) }
+ } else { + No policy. Set one with PutVectorBucketPolicy. + } +
+
+ +
+ +} + +const maxIndentedJSONDepth = 32 + +func indentJSON(s string) string { + if jsonDepth(s) > maxIndentedJSONDepth { + return s + } + var out bytes.Buffer + if json.Indent(&out, []byte(s), "", " ") != nil { + return s + } + return out.String() +} + +func jsonDepth(s string) int { + depth, maxDepth := 0, 0 + inString, escaped := false, false + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case escaped: + escaped = false + case inString: + if c == '\\' { + escaped = true + } else if c == '"' { + inString = false + } + case c == '"': + inString = true + case c == '{' || c == '[': + depth++ + if depth > maxDepth { + maxDepth = depth + } + case c == '}' || c == ']': + depth-- + } + } + return maxDepth +} diff --git a/weed/admin/view/app/s3vectors_bucket_details_templ.go b/weed/admin/view/app/s3vectors_bucket_details_templ.go new file mode 100644 index 0000000000..c74e739b65 --- /dev/null +++ b/weed/admin/view/app/s3vectors_bucket_details_templ.go @@ -0,0 +1,392 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 16, Col: 69} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

ARN
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
Created
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Encryption
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Bucket.Encryption != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Truncated { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, index := range data.Indexes { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if len(data.Indexes) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var10 string + templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var11 string + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var12 string + templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var13 string + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var14 string + templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var15 string + templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var16 string + templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index.
Bucket Policy
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Policy != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var18 string
+			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "No policy. Set one with PutVectorBucketPolicy.") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "
Delete Index

Delete the index and all its vectors?

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +const maxIndentedJSONDepth = 32 + +func indentJSON(s string) string { + if jsonDepth(s) > maxIndentedJSONDepth { + return s + } + var out bytes.Buffer + if json.Indent(&out, []byte(s), "", " ") != nil { + return s + } + return out.String() +} + +func jsonDepth(s string) int { + depth, maxDepth := 0, 0 + inString, escaped := false, false + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case escaped: + escaped = false + case inString: + if c == '\\' { + escaped = true + } else if c == '"' { + inString = false + } + case c == '"': + inString = true + case c == '{' || c == '[': + depth++ + if depth > maxDepth { + maxDepth = depth + } + case c == '}' || c == ']': + depth-- + } + } + return maxDepth +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/s3vectors_buckets.templ b/weed/admin/view/app/s3vectors_buckets.templ new file mode 100644 index 0000000000..626212dd01 --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets.templ @@ -0,0 +1,189 @@ +package app + +import ( + "fmt" + "net/url" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ S3VectorsBuckets(data dash.S3VectorsBucketsData) { +
+

+ Vector Buckets +

+
+
+ +
+
+
+
+ + Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. + Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine. +
+
+
+
+ { fmt.Sprintf("%d vector buckets", data.TotalBuckets) } + if data.Truncated { + { fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets) } + } +
+
+
+
+ + + + + + + + + + + + for _, bucket := range data.Buckets { + + + + + + + + } + if len(data.Buckets) == 0 { + + + + } + +
NameARNEncryptionCreatedActions
+ { bucket.Name } + { bucket.ARN } + if bucket.Encryption != "" { + { bucket.Encryption } + } else { + None + } + { bucket.CreatedAt.Format("2006-01-02 15:04") } +
+ + + + +
+
+ +
+
No vector buckets found
+
+
+
+
+
+ + + +} diff --git a/weed/admin/view/app/s3vectors_buckets_templ.go b/weed/admin/view/app/s3vectors_buckets_templ.go new file mode 100644 index 0000000000..8b486b043a --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets_templ.go @@ -0,0 +1,202 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "fmt" + "net/url" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Vector Buckets

Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Truncated { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var3 string + templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, bucket := range data.Buckets { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if len(data.Buckets) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
NameARNEncryptionCreatedActions
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if bucket.Encryption != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
No vector buckets found
Create Vector Bucket
3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.
The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and encryptionConfiguration, which the S3 gateway checks against its SSE-S3 key.
Delete Vector Bucket

Delete the vector bucket ? A bucket that still has indexes is not deleted.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/s3vectors_buckets_test.go b/weed/admin/view/app/s3vectors_buckets_test.go new file mode 100644 index 0000000000..bae344eb2d --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets_test.go @@ -0,0 +1,80 @@ +package app + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" + "github.com/stretchr/testify/require" +) + +func TestS3VectorsBucketsRendersBuckets(t *testing.T) { + data := dash.S3VectorsBucketsData{TotalBuckets: 2, Buckets: []dash.S3VectorsBucketSummary{ + {Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media", CreatedAt: time.Unix(0, 0)}, + {Name: "sealed", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/sealed", Encryption: "AES256", CreatedAt: time.Unix(0, 0)}, + }} + var out bytes.Buffer + require.NoError(t, S3VectorsBuckets(data).Render(dash.WithURLPrefix(t.Context(), "/admin"), &out)) + page := out.String() + require.Contains(t, page, `href="/admin/object-store/vector-buckets/media"`) + require.Contains(t, page, `data-bucket-name="sealed"`) + require.Contains(t, page, `AES256`) + require.Contains(t, page, "2 vector buckets") + require.NotContains(t, page, "No vector buckets found") + require.NotContains(t, page, "showing the first") + + data.Truncated = true + out.Reset() + require.NoError(t, S3VectorsBuckets(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), "showing the first 2; list the rest with an S3 Vectors client") +} + +func TestS3VectorsBucketDetailsRendersIndexesAndPolicy(t *testing.T) { + data := dash.S3VectorsBucketData{ + Bucket: dash.S3VectorsBucketSummary{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"}, + Policy: `{"Version":"2012-10-17","Statement":[]}`, + TotalIndexes: 1, + Indexes: []*s3vectors.Index{{ + IndexName: "movies", IndexArn: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media/index/movies", + DataType: "float32", Dimension: 384, DistanceMetric: "cosine", + MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"summary", "body"}}, + }}, + } + var out bytes.Buffer + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + page := out.String() + for _, want := range []string{"movies", ">384<", ">cosine<", ">float32<", "summary, body", `data-index-name="movies"`, "{\n "Version": "2012-10-17",\n "Statement": []\n}"} { + require.Contains(t, page, want) + } + require.False(t, strings.Contains(page, "No indexes."), "an index is listed") + + data.Policy = "not json" + out.Reset() + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), ">not json") + + data.Policy = `{"Version":"2012-10-17","Extra":` + strings.Repeat("[", 5000) + strings.Repeat("]", 5000) + "}" + out.Reset() + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), `"Extra":[[[[`, "a deeply nested policy is shown as stored") +} + +func TestJSONDepthSkipsStrings(t *testing.T) { + require.Equal(t, 3, jsonDepth(`{"a":["[{\"[",{}]}`)) + require.Equal(t, 0, jsonDepth(`"[[["`)) +} + +func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) { + data := dash.S3TablesBucketsData{Buckets: []dash.S3TablesBucketSummary{ + {Name: "vectors", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/vectors", Format: "LANCE"}, + {Name: "warehouse", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse", Format: "ICEBERG"}, + }} + var out bytes.Buffer + require.NoError(t, S3TablesBuckets(data).Render(t.Context(), &out)) + page := out.String() + require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`) + require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`) +} diff --git a/weed/admin/view/layout/layout.templ b/weed/admin/view/layout/layout.templ index 26f228ba4e..3e10120a89 100644 --- a/weed/admin/view/layout/layout.templ +++ b/weed/admin/view/layout/layout.templ @@ -189,6 +189,11 @@ templ Layout(view ViewContext, content templ.Component) { Table Buckets +
  • + + Vector Buckets + +
  • Users diff --git a/weed/admin/view/layout/layout_templ.go b/weed/admin/view/layout/layout_templ.go index 3da50ff198..8cb49a4651 100644 --- a/weed/admin/view/layout/layout_templ.go +++ b/weed/admin/view/layout/layout_templ.go @@ -534,365 +534,378 @@ func Layout(view ViewContext, content templ.Component) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var32 string - templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx) + templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/vector-buckets")).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 84} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 93} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\">Users
  • Vector Buckets
  • Groups
  • Users
  • Service Accounts
  • Groups
  • Policies
    • MANAGEMENT
    MANAGEMENT
    • File Browser
    • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "
      • ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
        • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if currentPath == "/mq/brokers" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "Brokers") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "Brokers") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
        • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if currentPath == "/mq/topics" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "
        • ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if currentPath == "/mq/topics" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "\">Topics") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "Topics") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
        ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
      ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
    WORKERS
    • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if isDefaultWorkerPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "Brokers
    • Default") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\">Topics
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
    WORKERS
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -900,108 +913,95 @@ func Layout(view ViewContext, content templ.Component) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
    © ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var51 string - templ_7745c5c3_Var51, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 60} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var51)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, " SeaweedFS Admin v") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "
    © ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var52 string - templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 102} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 60} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var52)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " SeaweedFS Admin v") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var53 string + templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 102} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if version.COMMIT != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, "(") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "(") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var53 string - templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) + var templ_7745c5c3_Var54 string + templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 353, Col: 55} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 358, Col: 55} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, ") ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, ") ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if !strings.Contains(version.VERSION, "enterprise") { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, "• Enterprise Version Available") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "• Enterprise Version Available") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -1059,134 +1072,134 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon ctx = templ.InitializeContext(ctx) ctx = templ.ClearChildren(ctx) prefix := dash.URLPrefixFromContext(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var61 string - templ_7745c5c3_Var61, templ_7745c5c3_Err = templ.JoinStringErrs(title) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 386, Col: 17} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var61)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, " - Login") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var62 string - templ_7745c5c3_Var62, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx) + templ_7745c5c3_Var62, templ_7745c5c3_Err = templ.JoinStringErrs(title) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 387, Col: 58} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 391, Col: 17} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var62) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var62)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, " - Login

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\">

    Please sign in to continue

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "\" rel=\"stylesheet\">

    ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var66 string + templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(title) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 405, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "

    Please sign in to continue

    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if errorMessage != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var66 string - templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) + var templ_7745c5c3_Var67 string + templ_7745c5c3_Var67, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 407, Col: 45} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 45} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var67)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\">
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/s3api/s3vectors/execute.go b/weed/s3api/s3vectors/execute.go new file mode 100644 index 0000000000..9a2a2e6872 --- /dev/null +++ b/weed/s3api/s3vectors/execute.go @@ -0,0 +1,54 @@ +package s3vectors + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +// Error is an S3 Vectors error response, as Execute returns it. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (e *Error) Error() string { return e.Code + ": " + e.Message } + +// Execute runs one operation as identity for trusted local tooling such as +// the admin console, like s3tables.Manager.Execute: nothing is signed or +// authenticated, and identity's grants decide what the operation may do. +// Operations that reach the vector engine need the server's VectorServiceClient. +func (s *Server) Execute(ctx context.Context, operation string, req, resp interface{}, identity string) error { + handler, ok := handlers[operation] + if !ok { + return &Error{Code: "ValidationException", Message: "unknown operation " + operation} + } + body, err := json.Marshal(req) + if err != nil { + return err + } + r, err := http.NewRequestWithContext(ctx, http.MethodPost, "/"+operation, bytes.NewReader(body)) + if err != nil { + return err + } + if identity != "" { + r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), identity)) + } + w := httptest.NewRecorder() + handler(s, w, r) + if w.Code >= http.StatusBadRequest { + e := &Error{} + if json.Unmarshal(w.Body.Bytes(), e) != nil || e.Code == "" { + e = &Error{Code: http.StatusText(w.Code), Message: w.Body.String()} + } + return e + } + if resp == nil || w.Body.Len() == 0 { + return nil + } + return json.Unmarshal(w.Body.Bytes(), resp) +} diff --git a/weed/s3api/s3vectors/execute_test.go b/weed/s3api/s3vectors/execute_test.go new file mode 100644 index 0000000000..38f7508aa3 --- /dev/null +++ b/weed/s3api/s3vectors/execute_test.go @@ -0,0 +1,66 @@ +package s3vectors + +import ( + "context" + "errors" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +func TestExecuteRunsOperationsAsTheIdentity(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), nil, nil) + s.SetRegion(s3tables.DefaultRegion) + s.SetAccountID(s3tables.DefaultAccountID) + ctx := context.Background() + admin := s3_constants.AccountAdminId + + var created CreateVectorBucketResponse + if err := s.Execute(ctx, "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "media"}, &created, admin); err != nil { + t.Fatalf("create: %v", err) + } + if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"; created.VectorBucketArn != want { + t.Fatalf("arn %q, want %q", created.VectorBucketArn, want) + } + var list ListVectorBucketsResponse + if err := s.Execute(ctx, "ListVectorBuckets", &ListVectorBucketsRequest{}, &list, admin); err != nil { + t.Fatalf("list: %v", err) + } + if len(list.VectorBuckets) != 1 || list.VectorBuckets[0].VectorBucketName != "media" { + t.Fatalf("list = %+v", list.VectorBuckets) + } + + var e *Error + err := s.Execute(ctx, "GetVectorBucketPolicy", &GetVectorBucketPolicyRequest{VectorBucketName: "media"}, nil, admin) + if !errors.As(err, &e) || e.Code != "NotFoundException" { + t.Fatalf("policy of a bucket without one: %v, want NotFoundException", err) + } + if err := s.Execute(ctx, "DeleteVectorBucket", &DeleteVectorBucketRequest{VectorBucketName: "media"}, nil, admin); err != nil { + t.Fatalf("delete: %v", err) + } + err = s.Execute(ctx, "GetVectorBucket", &GetVectorBucketRequest{VectorBucketName: "media"}, nil, admin) + if !errors.As(err, &e) || e.Code != "NotFoundException" { + t.Fatalf("get after delete: %v, want NotFoundException", err) + } + err = s.Execute(ctx, "TagResource", struct{}{}, nil, admin) + if !errors.As(err, &e) || e.Code != "ValidationException" { + t.Fatalf("unknown operation: %v, want ValidationException", err) + } +} + +func TestExecuteChecksTheIdentitysGrants(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), nil, nil) + ctx := context.Background() + if err := s.Execute(ctx, "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "media"}, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create: %v", err) + } + var e *Error + err := s.Execute(ctx, "DeleteVectorBucket", &DeleteVectorBucketRequest{VectorBucketName: "media"}, nil, "someone-else") + if !errors.As(err, &e) || e.Code != "AccessDeniedException" { + t.Fatalf("delete by a stranger: %v, want AccessDeniedException", err) + } +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 83018dca6b..0f629d2cc1 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -382,7 +382,7 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) // of the listing, Lance and format-less legacy buckets stay. out := &ListVectorBucketsResponse{NextToken: resp.ContinuationToken, VectorBuckets: []*VectorBucket{}} for _, b := range resp.TableBuckets { - if b.Format != "" && b.Format != s3tables.FormatLance { + if !IsVectorBucketFormat(b.Format) { continue } owner, policy, err := s.vectorPolicyView(r.Context(), b.Name) @@ -758,11 +758,15 @@ func (s *Server) mutateVectorPolicy(ctx context.Context, bucket string, mutate f }) } -// isVectorBucket reports whether the catalog record describes a vector -// bucket: an explicit ICEBERG bucket is not one, while a bucket declared -// before formats existed reads as Lance. +// IsVectorBucketFormat reports whether a table bucket of this format is a +// vector bucket: an explicit ICEBERG bucket is not one, while a bucket +// declared before formats existed reads as Lance. +func IsVectorBucketFormat(format string) bool { + return format == "" || format == s3tables.FormatLance +} + func isVectorBucket(record *s3tables.GetTableBucketResponse) bool { - return record.Format == "" || record.Format == s3tables.FormatLance + return IsVectorBucketFormat(record.Format) } // errVectorDenied marks a failed vector permission check inside a policy From b7b2a5947f482b06085e59c1643e2ca20d3c60f6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 11:32:03 -0800 Subject: [PATCH 31/31] admin: configurable file browser upload size limit (#12348) * admin: carry a file browser upload size limit The admin server keeps the total size limit of one file browser upload, 500 MB by default as the UI enforced before, and hands it to the file browser page. Co-Authored-By: Claude Opus 5.5 * admin: reject file browser uploads over the size limit The 500 MB limit was only checked in the browser, so any client could post larger uploads and fill the temp dir with spooled parts. Cap the request body at the limit plus room for the multipart framing, and answer 413 when the files add up to more than the limit. Parse the form before reading the path field. FormValue used to parse it first with the default 32 MB memory, which made the later 1 GB ParseMultipartForm a no-op; keep 32 MB explicitly. Co-Authored-By: Claude Opus 5.5 * admin: show and check the configured upload limit in the file browser The upload dialog reads the limit from the page instead of a hard-coded 500 MB, lists it under the file picker, and skips the check when uploads are unlimited. Co-Authored-By: Claude Opus 5.5 * admin: add -uploadMaxSizeMB to set the file browser upload limit weed admin -uploadMaxSizeMB and weed mini -admin.uploadMaxSizeMB set the total size limit of one file browser upload; 0 removes it. Co-Authored-By: Claude Opus 5.5 * admin: note why the upload framing allowance is enough Co-Authored-By: Claude Opus 5.5 * admin: test an upload whose files only exceed the limit together Co-Authored-By: Claude Opus 5.5 * admin: remove spooled upload parts when the upload handler returns Parts over 32 MB spool to temp files. The router passes the handler a copy of the request, so net/http never sees the parsed form and never removes those files. Every large upload, and every one rejected for size after parsing, left them on disk. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/admin/dash/admin_server.go | 19 +++++ weed/admin/dash/file_browser_data.go | 4 + weed/admin/handlers/file_browser_handlers.go | 47 +++++++++-- .../handlers/file_browser_handlers_test.go | 73 ++++++++++++++++++ weed/admin/static/js/admin.js | 7 +- weed/admin/static_gz/js/admin.js.gz | Bin 15636 -> 15540 bytes weed/admin/view/app/file_browser.templ | 5 +- weed/admin/view/app/file_browser_templ.go | 44 ++++++++++- weed/command/admin.go | 5 ++ weed/command/mini.go | 2 + 10 files changed, 189 insertions(+), 17 deletions(-) diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index eadd89b3c9..fd9abe2e0d 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -175,8 +175,14 @@ type AdminServer struct { // s3PublicEndpoint is the client-facing S3 address from admin.toml // (s3.public_endpoint); discovered S3 servers are the fallback. s3PublicEndpoint string + + // uploadMaxSize caps the total bytes of one file browser upload; 0 means no limit. + uploadMaxSize int64 } +// DefaultUploadMaxSizeMB is the default total size limit of one file browser upload. +const DefaultUploadMaxSizeMB = 500 + // Type definitions moved to types.go func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer { @@ -217,6 +223,7 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste icebergPort: icebergPort, lancePort: lancePort, s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint), + uploadMaxSize: DefaultUploadMaxSizeMB << 20, pluginLock: lockManager, adminPresenceLock: presenceLock, bgCancel: bgCancel, @@ -1574,6 +1581,18 @@ func (s *AdminServer) GetS3Endpoint() string { return s.s3PublicEndpoint } +// SetUploadMaxSize sets the total size limit in bytes of one file browser +// upload; zero or less removes the limit. +func (s *AdminServer) SetUploadMaxSize(maxBytes int64) { + s.uploadMaxSize = max(maxBytes, 0) +} + +// GetUploadMaxSize returns the total size limit in bytes of one file browser +// upload, or 0 when uploads are unlimited. +func (s *AdminServer) GetUploadMaxSize() int64 { + return s.uploadMaxSize +} + // normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning, // a value that is not a plain absolute http or https URL, so the file browser // hides its URL actions instead of copying broken links. Checking the string diff --git a/weed/admin/dash/file_browser_data.go b/weed/admin/dash/file_browser_data.go index 37385f5fb4..7fd14a3e6b 100644 --- a/weed/admin/dash/file_browser_data.go +++ b/weed/admin/dash/file_browser_data.go @@ -53,6 +53,8 @@ type FileBrowserData struct { HasNextPage bool `json:"has_next_page"` LastFileName string `json:"last_file_name"` // Cursor for next page CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes) + + UploadMaxSize int64 `json:"upload_max_size"` } // GetFileBrowser retrieves file browser data for a given path with cursor-based @@ -240,6 +242,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str HasNextPage: hasNextPage, LastFileName: lastEntryName, // Store for next page navigation CurrentLastFileName: lastFileName, // Store input cursor for page size changes + + UploadMaxSize: s.uploadMaxSize, }, nil } diff --git a/weed/admin/handlers/file_browser_handlers.go b/weed/admin/handlers/file_browser_handlers.go index 085679b768..fb5f122b72 100644 --- a/weed/admin/handlers/file_browser_handlers.go +++ b/weed/admin/handlers/file_browser_handlers.go @@ -4,6 +4,7 @@ import ( "bufio" "compress/gzip" "context" + "errors" "fmt" "io" "mime" @@ -247,25 +248,46 @@ func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Reques // UploadFile handles file upload requests func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) { - // Get the current path + maxSize := h.adminServer.GetUploadMaxSize() + if maxSize > 0 { + r.Body = http.MaxBytesReader(w, r.Body, maxSize+uploadFormOverhead) + } + + // Parts beyond 32MB spill to temporary files instead of memory + err := r.ParseMultipartForm(32 << 20) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) + return + } + // The router hands us a copy of the request, so the server cannot remove the spooled parts itself + defer r.MultipartForm.RemoveAll() + currentPath := r.FormValue("path") if currentPath == "" { currentPath = "/" } - // Parse multipart form - err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads - if err != nil { - writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) - return - } - // Get uploaded files (supports multiple files) files := r.MultipartForm.File["files"] if len(files) == 0 { writeJSONError(w, http.StatusBadRequest, "No files uploaded") return } + if maxSize > 0 { + var totalSize int64 + for _, fileHeader := range files { + totalSize += fileHeader.Size + } + if totalSize > maxSize { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + } var uploadResults []map[string]interface{} var failedUploads []string @@ -327,6 +349,15 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) } } +// uploadFormOverhead allows for the multipart headers and the path field on +// top of the file bytes an upload may carry. Go's multipart parser accepts at +// most 1000 parts, whose headers take about half of this. +const uploadFormOverhead = 1 << 20 + +func uploadTooLargeMessage(maxSize int64) string { + return fmt.Sprintf("Total upload size exceeds the %d MB limit", maxSize>>20) +} + // uploadFileToFiler uploads a file to the cluster via filer gRPC + volume // HTTP. This works whether or not the filer is running with -disableHttp=true, // since the bytes never traverse the filer's HTTP listener. The multipart diff --git a/weed/admin/handlers/file_browser_handlers_test.go b/weed/admin/handlers/file_browser_handlers_test.go index f02fead273..179be2a6a0 100644 --- a/weed/admin/handlers/file_browser_handlers_test.go +++ b/weed/admin/handlers/file_browser_handlers_test.go @@ -1,8 +1,12 @@ package handlers import ( + "bytes" + "fmt" + "mime/multipart" "net/http" "net/http/httptest" + "os" "slices" "strings" "testing" @@ -143,3 +147,72 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) { } } } + +func TestUploadFile_RejectsOversizedUpload(t *testing.T) { + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(1 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // 1.5MB and two 600KB files fit the body allowance but exceed the file total; + // 3MB trips the body cap. + for _, sizes := range [][]int{{3 << 19}, {600 << 10, 600 << 10}, {3 << 20}} { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range sizes { + part, err := mw.CreateFormFile("files", fmt.Sprintf("big%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.WriteField("path", "/buckets/b") + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Errorf("sizes %v: expected 413, got %d (body: %s)", sizes, w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "1 MB limit") { + t.Errorf("sizes %v: expected the limit in the error, got %s", sizes, w.Body.String()) + } + } +} + +func TestUploadFile_RemovesSpooledParts(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(34 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // The 33MB part spills to disk; together the parts exceed the limit, so the upload stops there. + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range []int{33 << 20, 3 << 19} { + part, err := mw.CreateFormFile("files", fmt.Sprintf("part%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req.WithContext(req.Context())) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String()) + } + if left, _ := os.ReadDir(tmpDir); len(left) != 0 { + t.Errorf("spooled parts left behind: %v", left) + } +} diff --git a/weed/admin/static/js/admin.js b/weed/admin/static/js/admin.js index 5de9b8bd64..77681f26fc 100644 --- a/weed/admin/static/js/admin.js +++ b/weed/admin/static/js/admin.js @@ -1180,10 +1180,9 @@ async function submitUploadFile() { const files = Array.from(fileInput.files); const totalSize = files.reduce((sum, file) => sum + file.size, 0); - // Validate total file size (limit to 500MB for admin interface) - const maxSize = 500 * 1024 * 1024; // 500MB total - if (totalSize > maxSize) { - showErrorMessage('Total file size exceeds 500MB limit. Please select fewer or smaller files.'); + const maxSize = Number(document.getElementById('uploadFileForm').dataset.maxSize) || 0; + if (maxSize > 0 && totalSize > maxSize) { + showErrorMessage(`Total file size exceeds the ${formatBytes(maxSize)} limit. Please select fewer or smaller files.`); return; } diff --git a/weed/admin/static_gz/js/admin.js.gz b/weed/admin/static_gz/js/admin.js.gz index a4ebc6fd53e1c820ea2b4eb2d59903db97544b99..d6a6220718b75962a8d0838a051246e1eb26a8bb 100644 GIT binary patch literal 15540 zcmV;lJWInLiwFP!00002|LnbQf7>>)DE!`^0@Y?i=|+;>WYc|WN9oV8lQy?;k`p_7 z_B?4$43Ut8nj*LaBwO|RyWij74+(%EB{@#F+ihO776=RmgTY`hGZ^gdqEmt|2}xd@ zqGQZwV}^?a{a?Jqr*T1Zv9q(gi{8I`h4O+-=oOlgG$#d`umbVLIH3iJ1uIr4A$iJH zbCQXj$s&sdWf>Y{PTpcM>*iR@dgyuwA%qGd7Da}-mo!V*rkvAZTUkN#`;GPOx2>#!6yB>zTYUXy-GnDu~Wn-M8 zB`zo)r-bjMM4<5^z98a^9g~y@f<`Ec4rJ|@85I<#^cRAXa%L}QBtyqZh5Jliql9E`EQTt36+pTn;IVmVhXp9OnDF~lc z{T(g@d#4Khb~qMvNl=EDbSkIGv^ta@UYF&84*6m{C)omx$qX+kE2<7p)t6`bL!j-P zjbYw6O~@D*0T3s$!m0Y+Ef-3Vu;t{0sB6v5j9tp50dj<935~H}1!5U0$Uhc@3q|a5 z)r?uP3M`$Om|yLW$%GZ;ooeSS9neP^ln1XBj8Yp zG6^iuYOUy=mbfqt&I&r6k^=BWBlO|pkf$^z!x*P&w;b9(7Nl4yvA~MMH0?$o5-jjw z%m;!^rzshAG`P;ksMkC2JJHbVT@L3s?^eWcD?Cqa%ILi%BbR7wMm^Lct2gv%rdJzY z2sY4SLGJL?wgCh37LX$@s^z1Lytc3yixqmbhd7B@mXu=dWhO|m#A&y_uQUnh60$3# z;UBQmP^fWsqd%232eI14qCSfB03%HWCB5Z;`{>c$-rk-=mYU>4RFJttinsB!l{uv%nz`HE&29F0&5lMRhj9Al@RI259wD=#PLl0*eTEJL`Y z5NL9JM6Pn2DKeT5x}#FdYno?We>dvYSB3tjk+N6gl!)gkfj^(EUMAhht!dLi_U(N2 zXT$yK&kHC9<+`>Q=)F`)73W|W8B%+V(SEtWg1G(2F{xTgd2N(0_6~-NDG{U2pIkIM zAMHV!f!e_RVZuNAdw`2#mZmu8B=KR_+Hyxm!EtDyct9ReB3O7nj2oXdcY>UiAvXS* z!~$`_3R34U=j*0SrWy@oqadfL3a9XW18%F=_BpOz!=H|Njo&Qi)lvODao^iQol}ft zoD^)1G%g_H<$*m;X?y{{SXNe40>4RGZJXK5Voay)e`3ua$(4lhm1a1%1tTh&R~ z0QvVy&gv`(kms0Y<;JX5Qk`$lWci(JN!FgLpgIEA{%||OlA)&PF`3{+D!RUuR0HFm zqc>DwD1WYn3mO@imiU5`4cI{&1h&!x4TO+b1PiO4`r5a8cSc&8&ioc)M!CO}v`bcK zutuWFQ$|gshV)ZCAT{0%hU4sXYsRwKMzO(g)@%@^NwH}uIXXyfJz1jW7Hg_)v-LEK z`r4{}{8z89c1`JzHTK6E)5p^j*U+pM6#}JyjN^->V0qJ#)w4i5rXK^Yg+q0f6N@!f z+p^#Fji2eN_B#Dae*>4=R|Zy3+X)?|2aLO?C8FIP8a=I@PsZG^FB(|}MpFogz0A14 zSxmaRWA8*#LGaf{^<@aCbn8ZhN~t(X8LwtZs>UUqC|7DSQ9NnNTSSmG@?L8n7}O@I ziCMj%F3YroqF0}DCI59|RAE5<#HFsepE1#`nV!u`>U``) zy}Q&mYlg20>Ux8}e;-YWIAZgBAxLs6Rw?nS^o9xLd5Tx)tI-HW8Oul%5}?*pb=R-z z((3f8<|#4w+wP#C@5NcjRr{9JsNFJyva-D%%R+l}W#NzmZAw~4l4OY9rUY|>M6pr<5W`X(Y4Xykp}rT&7v8V{oK7`Hmi6_hAkyd~aVBcw zcWm1ZmOiX<0%BQTb4WKz=+f|mtg7bbwT!Dkk!p=oI?V>ssxgR3CeMoGN(?S%R1h?t z4icPANrA?!NJuf*i)?tVHc+sXfD(XHQi!P6Tv4hGDP5}CoRh+~V>hs0V$7}}0S$nOy-KnoHPyxxjp=|~#VMZ4 zGxLZPvGeXUfjP|mpY5Dg~uiU6td!5$)6GT1}; z;88PBC3AmynI+^3;C{XL__0;n^V{b+$&#ZPO_Q!i9Y;!gebk!yf>0LPIxj!dg2Z=Z zV=hV%N@+Y(i*v>79tpFDM9icM*hlVu8jhLAVMkQhI(RBqy@QGpNv&)%qwQF`eU-qY-*& zx{_7|_$;F__KYn9MSDoLjU1X>pb`3d@8Mr;lS(p2qYCsBw1>V!duYEbI&hBhzd%pu zL!^NBQS{?8`0t7Q@AvY*XY#*qpG6;?z9|}^6D(%KNy=E!m4Bves<7{&T~wA_^vopa z!Q{B$ya%Du5sJ_Qh3^OY(Ib1dn2$-p)gBVF z`5bef@ULZ+Znl_vfy&upJ{0U!KtOf3H!Mh=;+S-Ie|pyaZq)rLxqfuhd(i#vtDlly zZ+F^9oqne}Sm`iv)0JSlH<O^Uz|a`ZespABUi!+HIth? z&Q?f@e1WH2+U(+#;0!I4Ukf%edsq&%YE(}?{O{9`|K}+TB#P|O@&~dEMLmctn^z~Q zBbH5QF_$4?4B+CFO)c`q>iePmW%m6(g5W_gPT717^k5Sc90se?00&-=3`oZ`Qm-87 z<6tLf#`O-cHjEyk9A!)(yu>txF!QL2akit-Wm|npE_O6FbtyvXL7ff9XdA9PK&ubB z_xj2BFhh%sT;(zxTQx&0mN6KKJQYiKq_R}ALR2ST$#_%M+kz8=dv*O`gsy{q_RCUN zsZ-^a9J345`s$kzS1hq}Q;XAAep*e_V6>_i2XASSkI*w2lZI5pyg8wp@5d%MUe-~1 zfES~?wX+6TmkgEVawxSCF1pcB#o-TBSA($#gy!ju%LIdL2IL?J0>0}_UBUAr7?m=l zz(AyB01nKFqK_hBw}R?eqwNU3|5`AC(Y&O!L(&%F`~%wEmMB)^FGh?iWK|7uq&B6O zndmly)GY=QK^WOT|B3cOpk;;U8OW8S6-hNzqH0DxB?zU)?}CNZC=oqj2~Mq<>pR2f zJ3R*j6?dGOg~cnKS(I}VRBsId_SZJOxU6Jr#ATs9^JQUR{mK48^HL%~gh{0_Q0yTFK8J+O3=Dj&J_c(7PKkgR zSiW`=OvPZxnDjmZN@2gP1EAeVzt&~C29{;>-XtcVdAe2699F|lc-?I>(RTeB3wv;C z-0Xe-{JEyTkj>}Ys(kx7w}9Nqytl&JaPG@U>P4=^)3Ura=Q*YQJ9U+spmhY0{dV)v z9`Z8F7ov^aRaKiqJODO{y-xwYpimz54Qk|B*}|tnK;1zvO5cX%+^LH{TdZTg%3|a> zeU!ID>jF|IGoz3{n!-~Q0Vir_P&L&Ke&BsR#scfp`V0ND3*~MAfb3pZt!M}{w_iem zIq61fC-0Ya3NocE^J}TnNbb$if)5B7K)#Op?%MsNw_Lxq^U>_bEUPt#gXn&xGK7FY z6{92g%@-LJ;jZC}M*iekWO5MhPp}j1qyKyQ z=Jk-vqc@$bx)m?<9Jx>*^CA`Hhz$vb(5n+f1?#l@$}R$tqxyhTQ?(9LQi$%kUVEK; z*A;3*V8fe^UT}#qSx**ey87z8k0MeN*|j)`KKlBxyuH@CC4ue$N7{y&RV<`V-g2$IkbY+*m@7!Z@TDBGfm=T) z#v-1ft~$Rr+Z`k#3NEE`dwBUj%y@85V zH?nnw`J(c_B9^5ql(I`w#F!J55+O*z`^MVXM+p%1RfR3i`Z)O8*=A2D8z~A3qme}j#4)-!8gzjW7^FW36vatM$A|nON z%<07@b755CQD1~tj8WN=15kvyYtL6M=-ZR>&*@6<4dmwFgXo zW_ux&XBkR@(aezH1&z>QQQ*}Oq653KZXj@0fToQ^QLY9BAW;V{fFtuS6znn-5gn6M zx+e>E8G@bE?f^sma;HX*dzF_hO%J5Jxu7{B^IWXd6-ofTuW{cw0O9_Marf$#_6d%1FBY!*0;)MEABV@%lc;WSZL}$4%M&L zvhhj}Co~nL=u$!EFs<_c5PA&LU1ybH6@9` ze1y)qN}M*~+M&V+_pSl=V=PD)-1IL`-;~~So~KlFqqC^@VejM3Fy_nigPryO5nSRS z-EiGrsDTQ*a2r^ZD9DmsSd>^4X(`XkjR;YW@`5dCLJ~AyDRp2VZC>~xG zR1iBzK?&)WimJDvn2{`~D*>rh7*BM>a;gvF2F_QGzS0{?RRuJaIv`J5u}<}dlKV(0 zE1oK5s42-*VjQ^p>nmt&4R4$%$#6~Qlno_^1!-Bg9jJl}g3Xl~(`0B);81X2IZ!w- zMG06(J;jy1XLM(`|Y>i=2ziL zLSFXj>~+=&2Ggz#;4*L~EMAn!^(yF6GpWM80>`%Mq+%18T#@()Vumt#!7yL>N=?z% z{l_(qsGF**P4J`VENBS`J#qTd%X@&nuH`;dI1cj#pGiytY}*|7RLMjGr_Q&(9dTEa zWY!X_Rs>E{4V7HQBo~`gA5IS4l&Dbdy#qwi6BK*&KS1=s1Ah|CqL-6!-nnelE1dVYlX3>OLS zqd6-GnoyD^9Elmu0AjgTdkAxcr9&GPM0O*km^^7hEqMq-hi_jF(Ge~RO1LVPDFp#k z%+i!7F{Z3Hl{Ap%XgKGFC7ol5Oo_1+crgHaOuMdJ(o9+Ptqc?I-@Q5|xQJ(Oae?Q& z>*w?2@?%d0E>x|Z)I?>elViF_VTPmcs?Q?`>?c{w67v4tOSneQGC1m30F2DEj{Phr z8JzR`DEc%`aduIg=Bj<`ZgEsq+jhsWdf0H+(8{&dr|uCG;>k`$&po-Kzium2=Bn9t z@}$J%u4IWmCIWzW`l9RUVlpZ4l>APm#iQT6wD?b0!kYVkmE{XNV&m#nAGck+6Pj+l zb`2U|VCnvi6dbfN*zGHOk6hF^zObBl=urHj1pMyoLjMT@L#?=9MP9t;&En!czlF3| z@U4Y~#^G~h#g;Y=KuN9{y{7c7^fkt?g`w3pmgYoZM z7QC+oSN|2}0zv#iR@rWT4Xg7vVXN~(EYD5<~q| z15|%&q}7iSk(+4-iH>QX>hW+ut!u93;a>+0CM>QK@Uv0EM6^~F=~U6u0l+6%8$ zRp-*{nuy6Gr-NE>uiO7!1^8B-wgvfm&t%1ABs5TIr_Db%{MbXz#ITlc_L+YhL>c?W+2&y5z4gJydN+a%y*a+zpD<_t zKd;3fS2MPc?p;gRR=D!}>>^{And&pTIlr^uWUJeT0A*KiX~DZ8A!QllqeOuf?}rv% zBFakouhlyp`5%fF)J;M`g9mW;gcC7Zh{@ppJMd!&Wpdv&L4l>(2bHT(KnNdQWC@wj z%)L=;M<=ccOGGB~F0a47)ki97uce;BC32K>w}Ff?vL3}8A)VoAqij_G!w2bvCANpT zSk2A#9XQ^jXZl4>_vWt7Tdx9_lp9Z9B_~$&p2~2n)KgGSUH0InyfuNgspNTm%HGyI z!3M-0nA#-s%?nx!-& zgV6Q=s+E(1(9ahr3e0C{P6l7AM1hL9$czMa?_JlByy=~<*_v8V_EHPnI7?77N?@Jo zD+ncbK(gi~O%^z<7=`Mik#x2Hsy zSU7j1-Y{o*w^y23kt-GSHHI2MYdBw*h<%674-0};Y=QVf|8a>kVX4QRd)FnFH=XaQ zM6~COX z?HKN0CT6iX*Q?7?INp^%zVNDn@lX!KL}KPYlAkOJ`K(}03O^Ch?G>>K1}o^4W;lgg z9RQ}TB&pifheV<3ZX;pkD)N;pTomkbP>`HpfoKdje{!1n+7`{G!2X||I!4aNu1-^E zSS9)wT9r1)BXg}bxBZIw&`s}F)~GG7+glLfO13df1=Me^rn-?4){E-v zd)IEaH&TXndC$9(@560O>E(5Lysn?VzF6Xjd1rtQN`ZCVKppjF322^9LCfjJpF@z5 z&|fz?*7-B^LnvSwa)35qb5keVPUyL?chdG&MQsYEPYtpN_)%RcuhDtEnQwDbU)bE& zI@c-DM!%C%Nzi05T0=0ifP9+U*|MX9oi8$z4ssAdx-Sz#yI_31r+o%1-nM~&=Y1;=I?b@G zY6h64>VH*r)gH2-t^4Xt5Nv4Rrw$IQon?|Ot5u0Xg}{(zak@wdmnSu;8i&69^i$Mp z$Mr?k4M#D~GFGLm+C{8DKmBB7pjwarVN%{Fs#0ab{G)>CI}Ea~9Y-OOAIt&uLo@^_ zFqHp2dh;6o_x7Fq-{Jee!~b5te`O-f^mb4Is77y5M6maHPKqPUNw>F#B4;zoZR#V+ zE8JzKd_|=(K@0D(&Gjb<^H2#ZADB+q0zL+_Cd)#x_WAce)dU6_!doYKLFc$wwH`0* zF8mXQ^DTJD2)Dk~IpavDp}WI#h9b*A&E;=8S7p5yICJdVaw`&r;qxN@WDIV`j_>;AK6{CYm`l% zcL&=xulp@@%qHRJshN!uvRTvA8q}iF>ss`u_f3i(v|F2vx~vr4MqN@EHfl=-wzS}v z<~yrsizrY%b;j!^5d0R1W%?K_PsY-lzo|Lg%XsFY=8!`$p;9@)^h(4Fk}P7i#;y5>IGbF*B}@s)<6-w)bA;a1Nf$gEbJh@l2yzZC(**2N}cX@&4p zRJLd#@A(-tP?QAB<=tPBs7s~yuGQ5r_1KgFcY{(or(%fQnVFDF_0lAt<1{6Orq}se zCFy15EUnP6G)NT~k)Q#}nCcqk8vM=%nZkAd3Y+plXXwZgz^1tlu1!L((zz1&;DESyZA;M`$)jD~8xDu({{sEJ#A}TLe>E%g;{5x` zs~?2O^<&MB>AQM~z>ur_bTuo6iavp-5KQgn=a*3V%8(+PdT8ChB;lf*lOiUWe3Mwdm3y>I2NP>0|95e+^d+7gwW)?6THZzsq#{(4oIa&jtmE$~sYmjl7Y(r!| zmrc>VYp>mN^_!XAKkT*ZLMeG&MIFMkDDH4seCb)}_!aD;4ShsmUy)i7uBXTbzGi$k%>>@rX(38= z@Rk}!(R3`m<5PK)$vE_{`Fws0r`Gn)t#&8Z#%vsRdO0jCPskAV2Fu3qrDoIUVCPq3 z&Sv+Q1F+2jww^6(rwqH-%^jnqxuDIH?H(5OO4D0|irXy*XWmr3#N2ucb2A^`1}bFe zS*k7Eq@MlcVC%id*m>GFp0!LdLY`qq{j~Vlu6}TfY4f@}*I5PC>5wGmLgBsZYEQm7 z-_#M?WWKU&DwRi6*Sk6I9o*H7uhUx80$Zc8)L(%OnQ5DBU-nSTQ@RX}L@Tx^fL$_u z4GQ*!x>4-*Bxo2k9n% z936L;p~Gb7GMTufR|?)-X9=B974-NP;<}}C@%NgZ>znN}bA*#^010YdG_)q^hLjBS z7qV6I4QK?DwN-LfXk=z~n?$X0YArVSPs!%-g_C(0T)gNT(_Xq|OxURPx{G#gnQH1H z<${B?q|EGS4bRkj?`C2=w?k@mmG8g^J-T-t$beYlQ7Sri@A|?XT8`;C^mk-O`Zbwo!GyboY+&&Z|0WkOTx_24IT7rF0uD zqMY6>1NTZg)T8!{FLTCJ4NxI+QV3eV3jmeyS9?qXkS#9wI8DlK*5C7Sh3W=R zhXgL~xP5PYg~ujW@Y>Fu3H=)dBYaZ<4s)aOw{GhAZPXMYi^jt;} zUyv1|nH!cWU*1K5aOw47KUNIeK_I%g_6WWADPXv{Agg>+&npZ@3QswN2P_@*J z?c%LMx7#~t8MtEFGHHS0(^U7$WvI4GlsifCfo_%KVKQ|bxj+7-Y)}dxq_2E_Wt0@%vb{-V3+l}n>^_|&UqTx5e>WkDd07KLNpJk*a zU~L%VA{l_f81>|{v?rKmmFgwUIt9TsR=y1zS0?5Xv48#(ePsdkp7;g3pSfzH_EJTb zF`3~dHD0I>5fqjgo#6*tR)41iFNylW^5|nJz9n3BL(7)m2DSPMDjW2XyTS9mGW9oR z#=12qTXO$Er?Zq!XKGr}FF-yiedU|BluOb~NrGVTh>BL)+zlCK2Bik9KYXzXSY+$^ zuGAy|n3u^M+PhFV_OSazX;T5okPDWBEdfvE5vJ>I7-bLT2wm7E z@-oH|Fo$7uC8-B;a(m9>NQX&+IGfA-U@}CRkOW>pbnQeXR9`-nU{kzuFXUAsL4R(^ zr zgp!IDeRL%s*EC_gXg(f1Qh}pkfV0O>kEPHChC?$_STX)jc4v?U;S7N3Wysy3;Za~+Ocr>uGUbRXd{XX!!&LMH^v z_f2fS{VydG5&Td9e+R#enGkFq{)FR;sf=LVN5yoEyAK~f?xX5Id&6&gA%KOv_wz8n zLJ8(GlAu5B?frE;nKacM6gZ&^zK`~XkN*0&AoF^~Uk38gwtX3WRsXCv{e3h^$yNO` zPU$oo$ZP@oNHJ=?($5Pn=wvmBl}RXE(VNwBHP>FPjS7-tL6;!k)MbbUaa9v1n@J_tziP1dM3v!mjlkMLRa<~NCaqctG(tq)Z80)cP4yXZ|$%yoDx43ZOJ?PYhVvNs;}Eu6If534&WTGbt-^nxHJGo3aB zq61ucXQ_U7iHE|8l_Fe8J83dPT`BY^M;MK%K%$`W#Y#T|PE&1I;=KkO?Ag+R{Kr;N z$WT+FUAuW>0~^XUEcapf+-m@2Unv{80DKOy$e^<}YI}}i0Wk2?Z=`NyRdGqumQ%r2 z%Q!&E(C)eeaTCNrT?alU>>L=}x7Jne=<^Wj;yN@I$f|Z^v>2 zpAzvx|9Z9pwOKYEdMThK4Ig|q5%bmHYjp2gA_jXD1Q zhpTCVLEgYUt%t~_dr)vGsjm0d1q-KZpY+T2OX7XaHL&U zL-m%%Q~TGGU4ZM2u&z}%Gl|O`CC_`@;c$p^j;%=v=KuurTwM=sUGwIGV3h_{?U=Eh z66N-UyiOb2uh5B?o^P3HMuYGCk^*2?B=Su4Ao z*UKj9vKfUK&q;v?OPUZCz~oWA2OR1F$90sQGOyeDg41}AF;O3Wo=lqCNiPb)#T*wG z1E}e@PFNk2gvD1Ig1lr!QiFMw@*2#mly3?Xu8ybmVdkRFXt~%FC{JlZcnzw!szaS8 z8^QzuQ-k;mtuu9ji(1)Vf2kFTK(*;F8zKY!`nddDp9fJd0_jyNjCsAH40)(ctkbv+ z!CNNU&Z_wnthYV!M*gs^Qc~wU#>Q2ma|B`U=!@8F*g4G|{L}%F545XqJ1b>f&Gn5W2spXQkh^KUjW1mMuvk)K?@J!Z}1Z z6%-4C78w=gc~xR!M`SBxVNXOOZ`kq%^25LA{~g#JojgN=q47#Mu&$BPB4v*^08dqy z;Lq4I!o~ZHif+|s^{`bNAwY{ed1lcOy4i$+(3{f`Fmv-M?822nO*jAxR03}cul8bf z;XvIcB0lK=w^P$(w8213W!s+^VlVhtYt)`@pcZhmL3{hk#b-aIrPf-trdpHy8D5fH zr0l2!jR_Zuf4XPS`sn*-eZ&fM@~r0s-G!J+g0VyI#)_%^GntkzidXm1o?_R16rDVa z(2XDHAl>-@j*5sQjN+8w44pkw3(4#2>BwF%-v933*CYKi>NeKut;JWt*lmMRj_7Rj z5q-bSh}wtpS!1Y>|2~AgcHu@(XBG_s4ywF8uid5zk z$>quwW`-(%1Uyb&ZW%PvH3crFL>z38dIrj!6MnUmpwZSlk%N(}qxG2jMRKbPAsIVp{7J$Abkn((E%V(CF-eEM1FHMX`xo|MP;mnDK*dEeP*OLf3&Np z3Xj%R^u5`YeYQMX_xAn2IP+gQg$;qETjmU36L%`Ji^)jRrZ`QxG+^R1n?fk6MbM63 zZEn85ue=b|dX7EDzfktS#%;?rv6|!pOphGkT423G%DS{kzJ^sH1*#N(4j( zK@*&GBts3xe2`E+r{z%XsU3BsKm{}vm<9sZO{~qdhs~*lT=%X^7XQw~iIxP{+oHC> z>sjxieO>W2%l@Vto0@Xb6a;gYDd7dQuJS6&m7PSro6jHDgfY?1Q){LLS;^-*NWb?x z>Nr_fpv`zoUgX1J-St5tmzigPIN>4o4CwK>j6M<%b@M0dEjOQ9V!O-!W z6W6vOV;5=0Ng)oCpK%PAJxBWnuf5Z-PHtU0S2l)o3jP;?3&H=J z!lgCo$Jlk_`1Dcsym1$`;^=~YyZIFHA_gy7_;#FgzZ;S@HC$q?WKa*%_T5*{Z;oJ( zk9s$qu%ys9()n~SNf$KfAe;)%qh~t+XGDdZJyssT*5!Tf*teRgD&HK|PZ^jB>U3H= zWv#1&mzc)|%|%p?IZAOhU8s~pQzBmJUoQ$aKcU7C3}E}H9DDCz^J!LLjrI9$bZu@F zPD!%~8^pATQ)2B8!KSD3+}`LgMgNpJCrX6|hI$ha+MZjrNs{LUdHMwS(BNyxJGggk z{tUCuUnJ|ufU-%-E(fc9gcpJxfU}qI80Q4!Vb?6mn22K`V+Tq9{HXKw-rk=(ND+M0k!y9ErCK9Yqc*3X%rFAiQsi`S5=juv zl#N)uT|KPb*p{$S#&nr;HxSmo@P@gE}-PtBrl&KFU(~?bn^1#x!Pxu6@oCQRfEdgSd--YrOQzMJZP(5Np-7LRp_^s`-yUn zmeu9YU`6RLE$hLrHe9Y8s8>xufhskWt-@fVt3kdJGlw?1prQJdSyiX3sTz>Vqg+$f z*d|3|r3Vk}YI)vRQJ?EK0m^42O=~0@V?Jvn)-1OvHETqpSd3RzP5HA4CMoRsl|LJy za^XPbjnm36>BO$9em7LUU?Qb959op>*fCD+*;T(ADkm(yAcefxUbV0agUwdh;d%M9 zVI?h{m9z|3f8wHG_!b>d&jI>hHCyy`Czs$*>Z z_{I8rLix-B5|j_?1w%1jClB(qVxZvbR{v(p{z9x@>zgg|m0YL#?+q3$#7mh!*Df{- znYHV8y&%|D5KXC|(~K3Q#CS>tHYxC(MHPzeBH3~s)5Mc$#qunhls{-j%fGDzL=`xw zP>+rZA>=qRDp(;uMoZqMS~9si@-O6WmuvkCUj4iAFQn7oo_`@Y(CuuzaE!Mc3~!xe zUlsiSWG{m{+EtV>Cp(Mte=|;j{|LLe$M*k8cJtcMzm(m)*3lQXn^*lA8;fsSX|RbNeb@11Ot!%L`Wa|xr;*GojanknJQ0qCA+RS z)8Fy3iy>~=T5k{##RZ<{{MHN2W)+)eI@R?j;n@?byLtX}=oV{d!-i~4LP`4oHu|aC z&5=>^b}cS8+z=|aVFUPg`x)jlN>V!(!Ac$e#f0ezOUQ=uj;7?(oF(4c4%N7&M1249 zc-^cmsO&U)y?YpKD>S}e4}Q8K_)UaBSXBfC?Ck6nqSZzz6-ARCHnaqUNIxmPS3W`1 z@Wso-5N2IZRX}qQcAa{{>+6S2?6fI!&)A}vLcu^<|7<~a_6l`DnxQj>UeP7#g_-@9 z3?JNj&t9$L1fXmapW9S_fFh*-UeAzCGGqO@<26^ zwvq;@0xVx1KiL(t%^SU$OgItSwE<0WyB5#S@N}~l%}dOxv7UHOLc(JBfclk?oW`FuzwQ-DbBW~k(o zz*Bo`-eSacZaHswYihGOe6s8D&Bk&beU9lVxCKaWTS5|4S>)Gl0%e=~_t6`f`3RCj z^H7!EzsIg1@RVD7!gDjg@}*w>*(!xC8#K{KCJIU#=vHP(`EWs2eRWsxW3NSQ_&UnH z>kG2F392a)SpCehB|G1lHlp1vDx-%^m?S9Vd#%i6f*}=Prko{ZxfOLLELZOm7if-i z=Z0N$A8vo$$MgKay7Co0k)^3{3ZKftsax2Qg>ciwF78AfDE`ylqi+w&n{n2#0Igv` z@)XCUyZhn(lc$~N_zRUmTRN?3@yvGGxnz~$IgP_+@|THBBC4uLh@T*G5nj+5 zW}eO4c0F=YEMl=Jh1VCEp=Ie7eX0hbE_FazePCz6T`gVA9y>kA>Y)xKmqxXk-eQI_nOfr%PY4>I z1t%O?b!_yc+f}_n(1M?G6dD|@HbZGR9wAesUcnhf>F5nqblJu8Tmt9Jom}JgRM>e< zE3)(sS}R(8TcYC`OID~`hWO%@lvkK$Qk?e@XFyI=%Eq2Q)e^ww%mmd#GcB{W`}N9R zRUQ;TvBLa?bRPTYsM0MEQVl{VDett)-I9o^$!(a(ZrQ?0PBovQT%UJiv(4G70Xt`n zMMs*8YlN$@m<`2Zv_sCxe4N_X&mDZ$g< zgRnatxH{lZ4?tOd3D>#aaoZi{I#NMz-UXXWjX`?PH*lJpF20DX+)=7dNn8-|BUyE^ zOmOfc9HO`>bnlj13RGDmtU^_7lt!8y8sKQuq@kc4bSs+QNQ(oRUHy z`Cl2rwC1XWcUkFS=vst6NUAX-K>C;nOjBMRGSS(LaDw83BqS3QryS)4gUo$M(E$mj zUNToHlMxpTo@p4O9|_5kn2|Zm(xM>h1rU}+0`VCxoF7^WgPB_^hLmKtQUHw0qZ@-)=sYtkH2$gQ+c8+EIso zXyJ3)O$QYOCoD??RwNS!;S0pUUbaz6Z>214F2*t*`(KSCa%U4aA zN`o;~&2FzdnOU{Oe&v}h7wv{KTgri7Z)Qsr|7~YhzY%}L=5w4S9Nxk!5WyU@(tL&P zqa1TVdUrSNFx9vq*q?3n7QMkEIOS-f@?V&(Tw44qc*|H?E89v`-?F00)KXU- zUX#m`)Cxi=29Il*X)Ns_BbQ%Bb%0)n>er`>TWb<7IJN#YxLYa~I(yU_-9lbD+)%mD ziLSmX*DU%U&=x?CRp`q{HD{QkF(D@KG5JF;_(L!FLofJ4FZe?*_(Lz)MlX0xE>R`n zwPz&pbpvS}m;Nc+9SN8z3kc5i8$A`;cfy?rQPiOzX1SfNXZjQHUR*Y Cmr9-h literal 15636 zcmV+vJ?p|BiwFP!00002|LnbOd)qeBDE!^O0?}qe=|nGXvf2BoZKaoECv8sSBqw%y z@BO4XF+@TVYKq_zkZi^Ee?J!n011GgUL2>}?Y19M3j_v(!C)|$84LykbV~3gA<2tV zbd33Q#Bh^#0u|low=7uh5jFIVsSX6^PGA2`xx0Sg}M2$y2tR zkxcB2=UFT$%g_jO@)nC}JI7+$LDxG7Ayg1CFEZ4=q*=l)`=35NJ3M{<>Fwd!51&4v zUw=hW)IkrCY_Y#{1HB*3i-KgLm$DcG+;~dj3y$yv(~Jv*^Bn5)E~1l+6*QY5F(nq( z6-tzAXia!&v`k-pStY|Wf`-GPHz2^TgdA6JPX#Un|8Kfo4}D%Wb35u7N_?NP5l+zp z7Zi_D!go?4&}bfC5OKziNlFAkLli~(Q2S*@1;r`-Oi)tJ?B$eX==jYEq8z0RCnVWP zSUi^#*T+fnd_gktigH0RQnaJvHz!9d6S55IMqN}6xNXKo%YI4*YIpX%PXPD_oFyqK zxcSV9nCCB8G5ZOpGy!tBpJ+->aE2$OsI`Y$-Y69XMsp!ph8C355lyLBB9@^sO$izm z?2?lL<#<9Aeq+>jSt1(9nmp>5(aHbjd4dH|1U@Axi3LfXK`#c~o1GhpQ^jW?nrMiy z<^-DO=!dhDe;^`9hEh#)UX5OUHB-?vWWiV}XwE%qpXp+|mF_bq1!W11Q9;H9;nS+W z!?|GZRH5Gv$AT^h%J70tV46&;L-_EzEDvtoc>2wRygzP`ZUw44bKJZ zX|Vu1e6?-J0eK6^5f|0+(M4WcxfqKjdc2D`iCLDEV(w)oNU^|ayS}e93Fs2CE2M!z z#O$gs)HvJGU&@-jSnXm_7e#u2k*0!@-tcc1J>K2j-F3)PlYEE@GIK~#mw_6qm`U}- zQVh_rarfW1T4Z?fie?ua4bciF>l>-q&+wv`((EEKOVI%6k6G~?$5V^AvYt&f)#J1v zV>CqaV}C-#p%4Wf%>`*k@{3I>sc}%QhKz=2j8jf@uV%g!u;I`ixj#`B2vN+5H5G`Y zoCN1R>YxV?Pzk-%k)i=kW;CmF!zP`C)wXEBAFt?-KOQKpi*M6u`PX28UNb@V&^t0? z3zC4gnip)8k{K2>MsYglf)sLc5(@JsrdXg$%+ZuirYW6Fh1>venwHzEWJ|u|EgXf4 zk0MNSoMO&F%HSkvM~cs#MWKkavS|SW2(}K3n3^)+%9NT$P|1~-V{}QPg2=rNmlTqk zTo;k69A}D*=7a92)bg6Z86Y$P)QZ%U^z0R_8O!8a)AYL`++g3T1t6slrQ!UhKmUi!`5G1H04gV z@ACav?obtQf0*(=`+Ij=OpoA*V=MNM#A)AYwiR&D?@Db z3yB5df)%9BU(VM}nM^h6L8Bn2sS2m^`v%-rukCYOy@o#>^%}oh&a0#PN8-M>g*vAg z$v7_93~5}XjE4hzoYMG0{*v@{1b&mW+CYKumb9x!)FsoV8U<_-bxj|J;41h2Bn<}W zm}QYbxu%Kau}bsmqF?45mR_zzy`C`lSZMwueOd8s!E!utEDL5|at<#|%WxAi1{QCP zlns#osN}59f)er!)2!T>)k>=K?U@oHcd;c|d#-}&2weNa?FdWynxe;KjOVFn`%+R3 zjDLyVP=TTRxfU*HXkc36OP*{n&3J>rR(ha;5E6@n0XouCU;9??&PYqsSxrz*Dff4h z)sj^;p@+2{bXQ7u05%c2IQcbFSoH(JXsg>bT@f;OH+lM|^A2jjl(P6D z@->0OBq_Z=AT*I+XvC6bz}eynVBIY8+}2^&&cAr@zy|)~jA=@eK6;xH%n1_3QUyTtOLe5lOQ(kVUMOF9!vb(R)fidU*Q0`f z(FbrQYT|cn+YSaFRyhGN)Ylx+juN^s`~X$e+`NXk3KXf zoKiv1Xwpk?HX#KXu_7TwZ#S~xx!OR%QX-WAoRUJg9S78glrB_l&Pie0u^U@p!DckJ z)eNgX%`#H_aCY(v4bgeY?FR{6D4rd*lHNEaSFNXx7_%z?pq^x6uM$+GrrL<25$%zy zIK?wKllMeHaV9mF8R=B&sQ0T@*}2GiF>ICP zUwW!!-)PpubHQ53Y)1*@Gg^*KqNdiDtFwJ0E4=06OHj2d%%hbwAiDtv2 zveF81r6^p&b2KLxiY>C5;WzVpy8_}DDBc~9AIUp(`E@fmwGHbO4LK2`#f5iJxqfzHk?e*5; zY95YcJ)Orf;k;S4p3)gj;m}c&#Aq}Z=#q*l;tC3q^G37DZ!LRG_kRjsSNK9GVlSpe z{D%>`e+trnA3=I2V)H8s!odK&V8sjzG+GM6RZx-A^D7}ah8oXgg*H^{qJ+eBhEtA) z=usJFI!5X*`(d+7bH({OxlB=pS8amwtr5pAK!Eyw!gdg zbJFPyCSBC(wyJ}b4ih(B38s6K(|-ZNO>x7~=}(AU<*X1sUdNPQoXL9a#Qb$eu7ne6 z2Ae(3mIy??z!MHOyEr8{Lv!WVk{g*lEC*UOs)G;z_vy#~^;8ZditN$yhfs#1PG1ys zR-LGhST?4`3_`@1go{%)vB(>%?}zZq?E765<1`)N_`>FENe?zQ!C`WB8sL)GBLmVg zjnpd#`p~Pg-j=KHY; zj)yu*5Ab4iw|3S@)+K#qx$J{B!bLmkt2q3g>Z&&qfzUj?ahYI{&4BC&LCEiVQ&;f3 z2u7t0DKHtHkpLVyCyG9bgzXBdV~w^W`2NqF35;eXtsRoCAkIH!yW0}QYW&rRQH89k zAr951^fHrVX#%NR^Z-FQvR{8iyCKj};dv(IO0p73HB_Q%Mja&xLF0G9%GC&np0EU` z*35OCVRW6IgMo@W&dkci3(hRcxe2Pbh9LLXReEt*$<~O=LVM=R!od22{-x%nM1l&v z0{m)DVl^@aP2G?hu`7^t+EAibz8se|7)4+Jd1M-`7?xGt9#{a1J;cE0kg%EJY(iH0 z7_136B|^|_!q-lMsTd3y1Mj0mDeSj(0IW9BuXUNOfo0jeH;D;oo^Dk%ldEATyzVxc zXuE!mg*`YmZuY)^`CQY_Wb^s9D&Kz2Eg*L??+QIj|&&L?{Re4%}p?|hzxf=ig z-Rr6q4du-3l~ClIv?H~Xcdgq%6Us8bmMV>4Z;s}?M=%$?Z=$Zdc0cJY*Kh57H2X2j zYR%P(i3||MvKF#nE|sAn!k3r|G$tapZq`Nv zoKxo}N7q}tGa{xe*+bFWH>YRTIKkIOa6$^c7g+C+zA@XAhss;>JCe&G(u z)U_iw9D*gwJ@mg%-@NW~IC|6ZvR&~)$B_&5G0#&`j@Xc(FMD-@sN^~=zp{%!ImXgj%+Rd7gI7*805icSK5Fj>x78-|pDgGX@>{r(x&^1ANDo!^`74uFV)OS6fF3`c+K;P2uWy(Z$46&K!lA6-tV zk@Xd*I+AAl(3@kqEx>*Vn@_cq*7qQHFEG&&YL{%1_Y0iPr4WL8efVX6rW{Z~Lp1om zA8_yUZtvTE@8iS!gT9bQKBy1((BQ9rO0tQVqNhXjxK?;DM1Oy*Sy3)^pKSuEs7dV!cJ5iHusu3N34vmRSPazl#993w|$zJza%b!qH*>~+DzGO)( zz)B113vwm2{P!U3)Ww-=tykyQjXHD1m=a&cq_1e>SbKaW#|G85&5-w8TGTHXyH1K* z?7HXPt~)Rri0Sj@ym=o!e*WtD+4Jc4xdWC1XleTXZZ^PX&e?PW{NwFqiM>AVa4%Cr z=uY-B4^(I`3(K!4GE%_IoL+2@3!@T``XanyjLMcAn3@~gjIPD43ro3h6exL`vyYtL z6M+O+DE5=FTh zw3ARBxFj6NzfiEtP(*Z0QgBZe>@oy9soepV^~;?aJ?T_lwlv)bd2>N?L}s~Isw}@& ze7fX*v%jrx2FF5E|4FERwU&)nx<96=AVr%BGLzE^|GzgJ!Zg@H;4E8^LP$|@bo!Hh zTa7Q!5Pc{ie~4h`@1p39q^|r!Sr6bA!Yf4&Bu56WVm;EbKv_~h_J3hCYe!xC5Z~|j zB`?}c_`w;!WIf3N-C8M3biG*5XZ?dUZS{}#?9{))B$TDrKos#!!OvM_QTLc#W>U6z zv>l~vq}8HlDI2vv#C-9wi>|?4yoVw=!GoADqS7X&+Rcl!%B=DJ-77uNH*opq{kvCf z3D{|YSA?9DhHc!}X))$SYDyA=`4F9Rl{jt8wL^vX?q5sXkFg-_PG7K>r*BH{InPrn z+R<6m`LO%(rXTag`Tox801;f`0d6?6N4gDE*oE7`qC`O!?82hNyhuxVUT#E)a+DWr zK@*ao(Nd`cJ&51p{a1-i+MrztR1X$ILjI8*ntOdxLOhJ0Ys9O{GbCvK={@d z>=lb~N(>z<-+R6U&psSXQ=04SW)jn}c78+m8OwT{#Pfo{>4jVg%dHA|LnNp(c4T0* zg%xBVCAYpY%Zh&0&w~djP~5*Ps33Nbf=ioU&!fVL?`` z+m@_?3xds+8PlY1PT;a&-*TXE$rL4E9rXZN<5V&Ol;c7mHU`D__|1t6RT=S3)EF9~ z76gv9_CwWDGM0yuR`1(yzx_793ReU%xEV50(mq_5yxK?`HYW1=l%zIL9#+%kF!L#Lrn2I(Mu;Y$#0NcwsK%eT^f= z(;@n=ut&9Em)fcHp>{0a_$X0T{$`uXkG84&c$><9+otl9Z7TnL<;en+TrbdtFVMrG zp>w4XqlYL0wrL1GF}mf^=}+yLFS;nIG)%?rJn{r!l$nk2K9V6`G}nrf)zDkizuoci+Q(1uipXozg1R%QaQuX zch%=%gdQT2#VjH3-@QCyvz%q}sAB;zGSfQtvz%n|oZm&!r%{Sk4k~AwtM)5*i=(RA zwmXK^!-l(tR<5l+buaQzHfHk-lxr1CzIx}VNo)%!GR8W+6UskdK)&xTii}O}nDPq* zHw2=O`*oT3;p6^J-Lx$~`V@M%oscZsd~L<>NkD4pedCXz68aZNR;Y-cCbu#$_ zPg=|ImNrUfKhdM@mc97`<^;>#gB$C)Cs*{>ZDq<#HQP>}l$hL=EYZhA3E-W+=z2OI zj|)5@e^6=h_;)WY{u7q4=KkMg`N|!!arLT?+b-TQwAya%8Z^Gb(*21PTxw-IDCn$*wUr}5oB`B z=ryHprLQrD%?+)#u~b*A2IB;$|MMB9Y4=d2<@k{-yQcz+Iqz!0)qRDzK(E+@W|i&c zx3D_@J#2NJYrN5Ll`sk27$kC_WwUQKZH9p|KayqH^wFkGIZ(1}_juEGCh&c@O7?^r zrkD$us

    ma8uvTY#8I=5TuDA@=dGTx_=FXQ-7AkP(RfG)!!Ov^`k`OruB1nzNw|U z6cCn~3HaPKGMRW?9eYTD9LB8HIS#cQ&Eh(^iYz-uiI(?QwoCz8>T*~tiY`YXNwkR(I*_!5H7;-Z_-SzFm zjICqu;QqA>eSfEwywQ3^_2O^()639F%_7y{qn|L}Le%ug! zT)x@2MU=5`oNW#^-d!JTtal>_)oqL~_b1HR|Icgj$JLDOq5IbW+X`2HpIv0^GE;p< zH|KX2oNRU55TNYpEi3SDNJv=*`6y9f#rvVTmx!{G{%iG4NB)PR1$C29(BOVN#RVr~ zI2YsI|F`eQ5JGa_H9>);+6Osu%TU03!y%ey2^rJOy-{pOC$0%gEHm%&`s-U=q@wm# z)HAq5j*{*+fEXj|QOqII8J;%ERt0H-J&;aV;_5ILtGT(pBaip!nSPPey}7IN)~moJ z<;K%j$%z%cr!pKX^;9aSUhZvEzA}NfspNTm%HGyI!6u2_Gqp<=shXF{c6meJ)wsKA^{QUzu^&Jko0ZFT84ZkY)@D?@79*s8uO-ywy&fjOt*7bY(G)rkldZFw8RVyb2p`R~M6qrxZ zjP$-yi2}8fWJZG8_pfV6-gM5_Y)var_EHPnI7?77N?@JoD+ncb&@7<~n#^%pF$(uN zrCf?8@xvW1NnAnQepPy$T4koVJ;RpFRt@UG0~gB%9xEBqMx)V)pfnv%CR7hEhO5ym zA%YY$nqfiemF?#bS0NG_%_g|H?;hR;wL2bv=pn{3n;8+6P=r^a;VtoswZQ-;Bb6?qz1u~Pc6U2B_SaGpJ=)zx-=V1A??-44MbXWQpLTaHYIP=k;d3%}sq_bKlCjPKh@9os>#~ zCX3M;f|&*2X>MoBj`nxH%1k=QJsct_D$qj@jIZ~sp23Q@ZQvnis%Or=GrOr?Sv9hP z(A~sV8bWqUqp8*cs(i%Fi5$Xs*4)oGXe@Wbbm`s=eQ&YWX4<<+v`txsF?J_jKosb?D(WTDin?gF6-+In_BwpsoArO%QBo;HM4_tDR+%Evr?DL4`n{W^p=C2#1pzsK%jhKmY7zOb)mys%|)n zaVB+qld@`nSb={2*~&n*9{sy^Oj&vHjJ3MD7vh>tk{;qRX)_aj>jy+p$g(CLlxIUTWriR(w&O3MJH`kJ) zb^p5Rt08mcZ|W~^-Yqxsl^>-97xh$?TkJPFebj%158m05 zUiVw*m`%dbQ!^VSWV5EJHK=)|*R9Z>-Zv?FvfbKj)McgUHtLeXuu)qwu=~mir_Fa( z(H2pldg_eVO(6MOAeQN4Fr19Ro4=_!+{<|8q2`c7FriX8$>{~e3zE!Z(r)wlOhpgC zLgw=sdI&%JoPH)G4$2fN0N}k~> zjmneVUG%@;!6yAbNQtI^Kqr-vx*Wh$(|(N(N8s!@N&IUQcb^i*R@+oK8kt2G`?Aa=SLoE%i zV?(S5x`K7(W*v3yW#hlvm=|nPz+-^803}jO3|p$GahRGLy(Fhspl;$WVgu+Iyir~Q z>no9cQ=>P+g$qgba%v)c*I=XSxezv0WW9stm#eYRqCWPOuM=2=(-kji0+KM={cEUB z{k=sHh8eq*hm~D}xZTDLi>=_=B=jnsD^X9g2~uvV%FiW#wRxxC@1Oq*H1`s(HLCm7 zw9t$5A1AMV5F*#lG&@G`>J0)zuJY5>wCF4P1Rg;!wVNMas@@k;WK&P9``05}lyg$V zB!k!3b)rysSgv66ENM$f3@0Q(10;`_0;e62#Jd55VKY^j^VjqJO#*(cH!4(1YUyN>L+Y$yg8Wv#!Y`YeI}F1ic!L96;j(qm zw_{GHaKeKp9RRlZa;6fxt#VQ*&0LYV+20dvw29|u)fvxm1(g_p-^InMoXqnE7spy( z&=#jHt;p$yJ;ry z&dv&=R7Y;mIEtnN=^dZPn~cSw|C&$cw{T}|@77xF-rAUr!|pDJh2d;`m}(VQ^QC6f zXn*H7W6oyBmjiH>>uWt**3J`lubaC>OGCjbAGUjL(JM`F4Ju-{63Y5D)l1B+M=&?@ z?roq#h90Ea!a?d;PWHFndyIu=^~ST7DTc^1>8PI;AKTUUZ!v9Ncjr2vpgJ9r#9Sr3 ze_id#H|LwWV4DnAmNljFh3a}Y=PPuiEoXe4)}j{J8jYp?{%goU+hp~!hgzP}AuJLt z*}RbKlIaUja$l&M!){N4hC$;IE5w>HL!E8MTnSTH)v?@}(o{cXd$&V5C9sFWiNzTT zyCECN$=WI{t88RucAFHfFtrvN{HNsc_{wQK%IleK+>L1u4jB_LYQ5{C zom%BRNVq~d=QNvCIUVhIVqNdOn}hM(4yM&rzRR(g4)0$Favzp>l!}g>_R#7hkpA!| zN-otHJ@?aslFQ&IZ-4h(4M+msa4^0{``+5%+b<2NTNrZOHlEIx?%pxpc~55zk`h5V z19FN;(N1ooMU>OKWyoGhhkCr8@nTMyhMX5HCxxK(TL2RD+w$w3b5hmu@0On?MEFGd zs+?7Ivl)L?v!g-;tM*&7S_NuLUJKBuk?Xu`U9#Dr7OrXE>8deZ_)?Ea{jtT>9;Zp! z&H5WYu29|J(U8Ek9k=g|FYnmo3SQW`Gl5^f4)Skt3#7UP@+FZ~tL|OsYjCoQo7RfE z7Pq_|u--QM+r*>)n}nk`?{4Zrc}ZjSGDv}VPGUNyv3k{3XV@>e|H?c~iF$p#00YPm zx|;p?pSGGjhxp+OvP3j<15)A5-B6(P`hXuRdTl=tS6q93-uo0V++2`lzNzO`4hDoL zj4cXT2ua|j_?n?F5IVlf4(A4(X!5FH6LT>i#Z#QAMZ{IW29Y|>7Y9@Uc}<~1*fQX# zPEv)-YJ#ezZfqBC6}s)veqm?6n&(xz0>!7P?v+E}kxP_2N$+vH>>e8tB}|j1L?88< zf#eGT`nkEo<~d3VJW)YB1&FG29A?RJ!E!qRSru`LE~g|@PB4yeL6Ax= zv*V}7vm?7#U0DHr z0^TR*Ap!T(7mof*i*LP-biH2VT zt1nW=04$sS|12Xb0@nI5E|Q*97^4nEU!Gu^RjL=5bqa!Otb7|bu1w4&V!!^1?pXl6 z$9)F_^wwN5QG2N(%ZNhe#Hdxt!$>wy6G230@HOgXPi3QhWnkbvMhF-v+h% z3Mw0Pk-Ne3t}^vE=ES-+C|h#>Kqu3bPNr&F(Pv3MkiPJmE#*L(2}zI~JfdPHZSIB) zQ-e|i)*rsw1kAH_eOGFd1elj;99DOsaO`T`S1zdvyo#JPOV67(SG`%K0+K!#EPq?D z98cf~({?wEvWIenF6SLX-(fMmWB3?L;M1 zUp|yzQ@nJq;#r}r>hp?BdI??l;#W6P!_apmBd51;eJ9gaO|d>m=)y#Q3UVcSvAV>V z_nx$#2A%nSv}l~NdD7DkMu--Dd653&AhaP!e@Y4~y8mD> zebUe@ckrIg1xZ>@-x2NGbbX%kx0uRBPYal-<-s7Y!#x-zbm11ChpDPIshD#ej$Eg# zdGvG-;Sp!)Tm(WV1k3l(?tUOzWGsRo3d!HWFC!)dn}t8&_+kPPtb3@KjBxwWlP6tN z{j=Nuwi5zaz>S|r`4vhqpOOUqWq0?#M&of)-CltcI_G<6xBvLRo)lzOulTtK&$aDA z^i}<{-t_m-I3-v0&p4%%tOt1l_K;%KdZl0HT+s2d7b}xcxS}_!1#eyz; zrKJ*yR{BTH9Cho3C#fuHyjE7J%McCXswPf0lS;0C)nM(3D#Pg-fwhCGwg7ibTD6jk zguq+vX9dpq*q#yhj!vR) zbw@?3y1kTM5X2zIX+t2ogbTNo>bI77D4ba5xh|z0n2b;xgdXJxqY)KI6g0kA>IcB3 z&I+}7rvV3hwsau>u~if@)Rbt~Zr<3y`mly!ABNjD4S?*+WJ4E#&mk5WbiPJy&rvKe zWtV}ws2f>T!hy1(YznqwUT^QFPG~S)g%VFz(JW?2pvEe);G#;rA+;I^*JxC%I+EI3vM57djkFaTr*6GKvOYGA&aV|$|XIygZg`6Z$!i;A(_3A z8ij8*>OHbFz5^}_mQ9{E+~_E0?qHxRS}kXgBekxZ!5zarbpKj~q!mFE#otuLJ zG*zIKDrOSzxtIBXsNpAME#8h{1D_BG_maP!Ev4EFjr(5mXQ1K3FD7EX?0tjoU#ll+ zpCZU2sC6U3a-B@j)2jnL)*%OUaYqkV(bD}L+!Hat5-Ch+P@A460SGGx>nuHBrbQ9JnwOb!y(Q& zwk9Q<1CX5O>U!wPHE%8mRtZqmjv3qeprJ?(d_b-Q$+(jlI?Q^nYdPB03Br6cp-O#| zkFzh>D|TsenV4Z-Qsoc3auknPo_(TU@=3H;D*Y*b-AUndN`0fx!z}E1R)Jt?Y7EFB_+eW)xyPBL(g)XhK*3lgITQaHt0y*HL!L zyl!W6PUBw2M1A;qGHz}MUKE0h87?k*vZmiUVRcLr7GG@$@{$!v4dzwKYcQ`;z9~#{ z6_d0+%v{tNEf<>tF?Qd-e5MSAlvCf#>BL6OH;LRDM=Mv)ofv7yBE4 z(EUXnEAh7d!SVyLY(Wa4M=Th^d0w!*pi<4BXH=BuRlvlK$X3Y8JrNDPVFyw|n2sPX z1RBX^7OZQev`Faj1|DgG>Hz+XJtJJa&!}iueO6Cets;cf;!d7fbd=p}LP6P^(@)@XIQfm*=L z2JP)D7hn8{mRf7onp&CO&+rm(k+P!_G$LFm{%M~*>!Req@75{0k zlv?^a+zmo0tOR;q=%O-3D)R|&xpIY>p~8=V$H~hrgNC}Mz{P}!{S8vj$a3d|U+pAo zg-(4Zaxj8Ax=&AXeA1F{ugX(1Awq*SA0sv2krNB4YN#oJ3#2bI(GCHTRiX~0o!s@C zt4RyBl3!G=N;#$G;kz%4^yZHTimH5Y|GJ{@%>epb@nGHC_y6LYf8`W51d?u8q{R@ff_gI+>$$|efo z-c=g1J=WT2@-@W_)aonI+fZFc1-LE1^cWudb;;u2nK;ps;Cfrs7I;1D9kiz_zGm4^y0NJVi>4r$vrGvuN$V=3GOX+* z>fC(!xWbc^};t^v3C&CM|?h;d%6I2jG;bkh3Ss z1GsW|pF8%gW~#z#!}<{eQ$d|hYp1Mrb?^f7xS+X+>M=(t&L(q}a%e)tEB)(5!Dc7a z_<A^sDxe0UC`65ro(Os@tG8@Tr@V{);BE7A9V{9St&ktMQ?C$=xg%rVu zEm*7DEY%vJ8nrq7ScVa}mLjKvlSqPSrfkIO)#`rj#A4S9HgyfBvm(aFn`=W3s|Ll9)sdJIBM zlJAu+L-@JBs(vNatyWcG%{MIf6XhH&tHaM=MR1sw_2jQrxLi3bb=h2pBRP2#H7hgC_rp(9Uz;Twu3zL`A8m+OVVy4AnQvX_h33?h!J zKd-K;zt^tz)k0%hLYn6J%9^~Y3hUnd_-y?>rto8>d{{3SiqSfGkgpX31z)%NH(T}> zV*Og*Y>_Wvo$9|gShTaGV&Jul%|d4F`du#wwiQGZD(EC*1t~F}P=QSfd}mRGV!KGT zSjRN+cv7)E%f{son$hxaD*;gjPFAQ#M}-h_92r(7rhp$~Mc$=aGPyhQFXV2QYyAse z{k!omq?6yDe<3)~?QFdA7;iZk-a5&?D)|4&UIulvt0-emb{6OVZkz)D5q5Kr?f;YP z=Cz@JExUQGqpxf?w>BLuLVerH$_1!UL529ZNImR^5R~c+D^<|5Vu#Z}4?xIk4=Z>gsrpmzp*mb>`{*ISjoU+(nZ)bEyxWKcV-+H0htYXtl zr@Hm!A~{>zljhC ztBO(qJ3D)YXthy-qG-~?hL(U3=_jT4!V^RdU%X5VVb=9j1vC?3*QqDGzIoKdPMb3K zl+BBYEa-vt&lY57uTWb`Gjzt#E4m<^Ftgv1;e%W6*{gM&ASv6#=QhAcPoxnO{WAWPuIS9chJOd0e~vN^5yZtKukAp^kzKfL~Pea zYKq&ncz%W_o3&_OVpfgy#5;hOQ=CO+KfdFwk1BhrTy?b|^woOwE$1t713e%>r1kXv zHQ=E3Tbek#?Z$xJXcYyJ!8~hi{EHx&Oaa@#Sv6GhN#KdSHE%KEI=7rRyfwAi93Bij zzS&sLf05}axCMZ>Eg=c2Eb?nNfwIkm2j~rCKI&hPB@b2U{Cn&Q0#CTLCpvT^$b`QbtWuV?*bQShI8kJUGyN@L(u~~%lEA-U(o@S zrot(F3WXE5umy$xoHM7m6}4pXUv?jVyIK7zWaZK8S4-XEWwxW-NiERp; zbMkuqbWES(8mc+X;s@^v~5Sym}?(NuHWlCx?Uq*Ikgs{AjKLDZ}| z{z7HY2B%dmp4m=2mn<_pqjA_w{xX3iqN)l&{1l;w@Pbx9^K9O>>ye9M9*cQF^h;L9 zZ%({ffFMALoZsc~rW7@`-Grx=4>R(rQ8^YpD({o~db;s)MqL#3zNvNRRxX%&6GHhc zBlniDZUf%TZN?-U1cv21Mx%P82IO?lA@nMS+!R0anp^-u&1aF7GZ_4=H8(VrZqcV| z5b9C~l+_1zCb?@x7t<$B4^TbS0dQ$ltLZIfSq7;!PVtza9-4E)kyXb=58STm6@nG` zDMw+M53g#|2gC6YnHu#9&L~PpZ=j;fE}rL7a?Z@jHEvIZo!7J?OJL6IidNs2=y=AG zC2E%;zIX}p3eyb4c^7de$%#tY*zuUjHDKqgvFJ#1agA^_7PFyPjCRO5 znMqB$+Uy*OR^w^K!Xe>Tt16jgqlyZ*fWWWS)FAu~s8&z4@9c-&>5{7h{`3Hp#n*71 z>m7Hs!(2xy=*_!eGtd~od%l6w+;s6(T;+~ZbxPubh#$$ajUmCokMa=3O`&_Y+)|** z8etX63dI18G&wZD(XdHLE&C6(FVSpnUm@+vmU6X){rET~g+TBxgfOkSD&bvLdK9`A zp%0R3%t#`AOa!JWuMU~$Y)UvmaX}K235rvW@`5dBLXe_^A{(=MkP z7XWtL6&HWIT{gS0w0q<^@2b3f#dhEiKLA=xgxQi-gYF19(rS` zY8rZENzTj}!(6nZBbHN@7%@(1K4Q2??)da18w&5RUT1ppnseTgtfV}0eg z)-J;fI>CY!{jyE}vY>*Tkt;ocDxYhoFQz1G%q4k?u_!xMACfw*g

    -
    +
    Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. + if data.UploadMaxSize > 0 { + Total upload size limit: { formatBytes(data.UploadMaxSize) }. + }
    diff --git a/weed/admin/view/app/file_browser_templ.go b/weed/admin/view/app/file_browser_templ.go index 0554245049..f994bdd27a 100644 --- a/weed/admin/view/app/file_browser_templ.go +++ b/weed/admin/view/app/file_browser_templ.go @@ -852,20 +852,56 @@ func FileBrowser(data dash.FileBrowserData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\">
    Upload Files
    Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.
    Upload Files
    0%
    Preparing upload...
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\">
    Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.UploadMaxSize > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "Total upload size limit: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var41 string + templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.UploadMaxSize)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 320, Col: 67} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, ".") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "
    0%
    Preparing upload...
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/command/admin.go b/weed/command/admin.go index a4844f1b8f..59ac89bdb0 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -64,6 +64,7 @@ type AdminOptions struct { debugPort *int cpuProfile *string memProfile *string + uploadMaxSizeMB *int // workerGrpcListener, when set, is a listener already bound to grpcPort by // the caller. `weed mini` reserves the port this way because the admin @@ -125,6 +126,7 @@ func init() { a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging") a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file") a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file") + a.uploadMaxSizeMB = cmdAdmin.Flag.Int("uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } var cmdAdmin = &Command{ @@ -500,6 +502,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, s3PublicEndpoint = options.defaultS3PublicEndpoint } adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint) + if options.uploadMaxSizeMB != nil { + adminServer.SetUploadMaxSize(int64(*options.uploadMaxSizeMB) << 20) + } if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil { return fmt.Errorf("apply admin.toml to plugin config: %w", err) diff --git a/weed/command/mini.go b/weed/command/mini.go index 498635d302..2d963f7bda 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -598,6 +599,7 @@ func initMiniAdminFlags() { miniAdminOptions.allowInsecureBind = cmdMini.Flag.Bool("admin.allowInsecureBind", false, "INSECURE: allow the unauthenticated admin interface to bind to -ip.bind instead of loopback") miniAdminWorkerBindIP = cmdMini.Flag.String("admin.worker.ip", "127.0.0.1", "worker gRPC bind address; expose only with grpc.admin mTLS configured") miniAdminOptions.urlPrefix = cmdMini.Flag.String("admin.urlPrefix", "", "URL path prefix when running the admin UI behind a reverse proxy under a subdirectory (e.g. /seaweedfs)") + miniAdminOptions.uploadMaxSizeMB = cmdMini.Flag.Int("admin.uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } func init() {