Files
seaweedfs/weed/s3api/s3tables/manager.go
T
Chris LuandClaude Opus 5.5 4f24b756ba s3tables, iceberg: give tables created without a location their own directory (#12475)
* s3tables: give a table created without a location its own directory

An Iceberg table created without a location kept its files under its name
path, so a rename left them where the next table of that name would land.
Such a table now gets s3://<bucket>/<namespace>/<name>-<uuid>, stored as the
record's warehouseLocation and returned by GetTable; its first commit writes
there without moving the root. Records without one keep resolving to their
name path, so existing tables need no rewrite. Lance datasets stay at their
name path, where Lance clients and the vector engine look for them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: create tables and views in a directory of their own

A table or view created without a location took its name path, or a
unique one only when that path was already taken. It now always gets
<name>-<uuid>, so a later rename never leaves its files under a name
another table can take.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: load and commit a table at its warehouse location

A table created through S3 Tables without a location has no metadata until
its first commit; loading or committing it now uses the directory recorded
for it, and only a record without one falls back to its name path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: fail a stage-create whose marker is not saved

The marker is how the commit finds the location the stage-create handed
out. Without it the commit fell back to the name path while the client's
files sat under the returned location.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: expect tables created without a location in their own directory

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: rename a table that keeps its files elsewhere before reusing its name

Enterprise keeps the storage marker on a renamed table's files, so the name
of a table whose files sit under its name path stays taken there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3tables: remove the table directory when a create backs out

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: answer a create that lost the race with the registered table

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: answer a view create that lost the race with the registered view

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: fail a staged commit whose name another create registered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: remove the directory of a create whose name another table took

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: keep the metadata of a staged commit whose name is taken

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: write the lakekeeper table's data under its warehouse location

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: commit a staged table at the location of its own stage

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: never continue another stage when a commit's own marker is gone

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* iceberg: test filer honors exclusive creates and token-guarded deletes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 16:37:31 -08:00

243 lines
8.6 KiB
Go

package s3tables
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// Manager provides reusable S3 Tables operations for shell/admin without HTTP routing.
type Manager struct {
handler *S3TablesHandler
}
// NewManager creates a new Manager.
func NewManager() *Manager {
m := &Manager{handler: NewS3TablesHandler()}
// Default to allowing access when IAM is not configured
m.handler.SetDefaultAllow(true)
return m
}
// SetRegion sets the AWS region for ARN generation.
func (m *Manager) SetRegion(region string) {
m.handler.SetRegion(region)
}
// SetAccountID sets the AWS account ID for ARN generation.
func (m *Manager) SetAccountID(accountID string) {
m.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default.
func (m *Manager) SetDefaultAllow(allow bool) {
m.handler.SetDefaultAllow(allow)
}
// SetTrusted lets trusted local tooling (shell, admin console) bypass authorization.
func (m *Manager) SetTrusted(trusted bool) {
m.handler.SetTrusted(trusted)
}
// SetEncryptionChecker sets the handler's SSE-S3 availability predicate.
func (m *Manager) SetEncryptionChecker(fn func() bool) {
m.handler.SetEncryptionChecker(fn)
}
// PrincipalFor resolves the caller of r the way the catalog's handlers do,
// so it matches the owner CreateTableBucket records. A nil request is the
// handler's account ID.
func (m *Manager) PrincipalFor(r *http.Request) string {
if r == nil {
return m.handler.accountID
}
return m.handler.getAccountID(r)
}
// PolicyContext builds the evaluation context a Manager call's synthesized
// request produces: the configured local-account predicate, the caller's own
// identity actions so admin and action grants resolve on privileges rather
// than the forwarded name, plus the open-by-default fallback only for a
// trusted caller or no resolved identity. A non-empty bucket lets an identity
// action scoped to that bucket apply, as the handlers do.
func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext {
ctx := &PolicyContext{IsLocalAccountID: m.handler.localAccountIDs, IdentityActions: getIdentityActions(r),
TrustedIssuers: m.handler.trustedIssuers(r), TableBucketName: bucket}
if m.handler.trusted || (m.handler.defaultAllow && s3_constants.GetIdentityNameFromContext(r) == "") {
ctx.DefaultAllow = true
}
return ctx
}
type tableBucketCreatedAtKey struct{}
// WithTableBucketCreatedAt binds a DeleteTableBucket, CreateNamespace,
// DeleteNamespace, CreateTable, DeleteTable or tag operation run through
// Execute to the bucket created at createdAt, so a caller that authorized the
// bucket itself never writes into one created again under its name.
// In-process only: no wire request can carry it.
func WithTableBucketCreatedAt(ctx context.Context, createdAt time.Time) context.Context {
return context.WithValue(ctx, tableBucketCreatedAtKey{}, createdAt)
}
// boundTableBucket returns the bucket creation time ctx binds the request to.
func boundTableBucket(ctx context.Context) (time.Time, bool) {
createdAt, ok := ctx.Value(tableBucketCreatedAtKey{}).(time.Time)
return createdAt, ok
}
// otherTableBucket reports whether ctx binds the request to a bucket other
// than the one created at createdAt.
func otherTableBucket(ctx context.Context, createdAt time.Time) bool {
want, ok := boundTableBucket(ctx)
return ok && !createdAt.Equal(want)
}
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
body, err := json.Marshal(req)
if err != nil {
return err
}
httpReq, err := newManagerRequest(ctx, operation, body, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
m.handler.HandleRequest(recorder, httpReq, filerClient)
return decodeS3TablesHTTPResponse(recorder, resp)
}
// AuthorizeCreateTable checks that identity may create the table the request
// describes, without creating anything. A deferred create (Iceberg
// stage-create) writes into the table bucket long before it registers the
// table, so it passes this gate first.
func (m *Manager) AuthorizeCreateTable(ctx context.Context, filerClient FilerClient, req *CreateTableRequest, identity string) error {
httpReq, err := newManagerRequest(ctx, "CreateTable", nil, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
target, authErr := m.handler.authorizeCreateTable(recorder, httpReq, filerClient, req.TableBucketARN, req.Namespace, req.Name, req.Tags, req.AllowMissingNamespace)
if authErr == nil {
if err := m.handler.checkVectorNamespaceWrite(ctx, filerClient, target.bucketName, req.Namespace); err != nil {
return err
}
if req.MetadataLocation == "" {
return nil
}
tablePath := GetTablePath(target.bucketName, target.namespaceName, target.tableName)
if authErr = ValidateMetadataLocation(req.MetadataLocation, target.bucketName, target.namespaceName); authErr != nil {
m.handler.writeError(recorder, http.StatusBadRequest, ErrCodeInvalidRequest, authErr.Error())
} else if authErr = m.handler.refuseClaimedLocation(recorder, httpReq, filerClient, target.bucketName, tablePath, CatalogRecordRoot(target.bucketName, tablePath, req.MetadataLocation, "")); authErr == nil {
return nil
}
}
if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
return decoded
}
return authErr
}
// AuthorizeUpdateTable checks that identity may update the table the request
// describes, without updating anything. Credential vending gates on the same
// permission a commit needs.
func (m *Manager) AuthorizeUpdateTable(ctx context.Context, filerClient FilerClient, req *UpdateTableRequest, identity string) error {
httpReq, err := newManagerRequest(ctx, "UpdateTable", nil, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
_, authErr := m.handler.authorizeUpdateTable(recorder, httpReq, filerClient, req)
if authErr == nil {
return nil
}
if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
return decoded
}
return authErr
}
// AuthorizeUpdateView checks that identity may update the view the request
// describes, without updating anything.
func (m *Manager) AuthorizeUpdateView(ctx context.Context, filerClient FilerClient, req *UpdateViewRequest, identity string) error {
httpReq, err := newManagerRequest(ctx, "UpdateView", nil, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
_, authErr := m.handler.authorizeUpdateView(recorder, httpReq, filerClient, req)
if authErr == nil {
return nil
}
if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
return decoded
}
return authErr
}
func newManagerRequest(ctx context.Context, operation string, body []byte, identity string) (*http.Request, error) {
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/", bytes.NewReader(body))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/x-amz-json-1.1")
httpReq.Header.Set("X-Amz-Target", "S3Tables."+operation)
if identity != "" {
httpReq.Header.Set(s3_constants.AmzAccountId, identity)
httpReq = httpReq.WithContext(s3_constants.SetIdentityNameInContext(httpReq.Context(), identity))
}
return httpReq, nil
}
func decodeS3TablesHTTPResponse(recorder *httptest.ResponseRecorder, resp interface{}) error {
result := recorder.Result()
defer result.Body.Close()
data, err := io.ReadAll(result.Body)
if err != nil {
return err
}
if result.StatusCode >= http.StatusBadRequest {
var errResp S3TablesError
if len(data) > 0 {
if jsonErr := json.Unmarshal(data, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
return &errResp
}
}
return &S3TablesError{Type: ErrCodeInternalError, Message: string(bytes.TrimSpace(data))}
}
if resp == nil || len(data) == 0 {
return nil
}
if err := json.Unmarshal(data, resp); err != nil {
return err
}
return nil
}
// ManagerClient adapts a SeaweedFilerClient to the FilerClient interface.
type ManagerClient struct {
client filer_pb.SeaweedFilerClient
}
// NewManagerClient wraps a filer client.
func NewManagerClient(client filer_pb.SeaweedFilerClient) *ManagerClient {
return &ManagerClient{client: client}
}
// WithFilerClient implements FilerClient.
func (m *ManagerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
if m.client == nil {
return errors.New("nil filer client")
}
return fn(m.client)
}