package iceberg import ( "encoding/json" "errors" "fmt" "net/http" "os" "strconv" "strings" "github.com/apache/iceberg-go" "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // parseNamespace parses the namespace from path parameter. // Iceberg uses unit separator (0x1F) for multi-level namespaces. // Note: mux already decodes URL-encoded path parameters, so we only split by unit separator. func parseNamespace(encoded string) []string { if encoded == "" { return nil } parts := strings.Split(encoded, "\x1F") // Filter empty parts result := make([]string, 0, len(parts)) for _, p := range parts { if p != "" { result = append(result, p) } } return result } // encodeNamespace encodes namespace parts using the Iceberg REST protocol's // unit separator (0x1F) convention. This is only appropriate for protocol-level // encoding (e.g. URL path parameters), NOT for filesystem/S3 paths. func encodeNamespace(parts []string) string { return strings.Join(parts, "\x1F") } // flattenNamespacePath joins namespace parts with "." for use in S3 location // and filer paths, matching the S3 Tables storage layer convention. func flattenNamespacePath(parts []string) string { return strings.Join(parts, ".") } func parseS3Location(location string) (bucketName, tablePath string, err error) { if !strings.HasPrefix(location, "s3://") { return "", "", fmt.Errorf("unsupported location: %s", location) } trimmed := strings.TrimPrefix(location, "s3://") trimmed = strings.TrimSuffix(trimmed, "/") if trimmed == "" { return "", "", fmt.Errorf("invalid location: %s", location) } parts := strings.SplitN(trimmed, "/", 2) bucketName = parts[0] if bucketName == "" { return "", "", fmt.Errorf("invalid location bucket: %s", location) } if len(parts) == 2 { tablePath = parts[1] } return bucketName, tablePath, nil } func tableLocationFromMetadataLocation(metadataLocation string) string { trimmed := strings.TrimSuffix(metadataLocation, "/") if idx := strings.LastIndex(trimmed, "/metadata/"); idx != -1 { return trimmed[:idx] } return trimmed } // accessDelegationHeader is how a client asks the catalog to hand back storage // credentials along with the table metadata. const accessDelegationHeader = "X-Iceberg-Access-Delegation" // wantsVendedCredentials reports whether the client asked for vended // credentials. The header carries a comma-separated list of mechanisms. func wantsVendedCredentials(r *http.Request) bool { if r == nil { return false } for _, value := range r.Header.Values(accessDelegationHeader) { for _, mechanism := range strings.Split(value, ",") { if strings.EqualFold(strings.TrimSpace(mechanism), "vended-credentials") { return true } } } return false } // writeLoadResult writes a table or view load response. The FileIO config it // carries depends on the delegation the client asked for, so a cache between // us and the client must key on that header and not on the URL alone. func writeLoadResult(w http.ResponseWriter, status int, v interface{}) { w.Header().Set("Vary", accessDelegationHeader) writeJSON(w, status, v) } // writeJSON writes a JSON response. func writeJSON(w http.ResponseWriter, status int, v interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) if v != nil { data, err := json.Marshal(v) if err != nil { glog.Errorf("Iceberg: failed to encode response: %v", err) return } w.Write(data) } } // writeError writes an Iceberg error response. func writeError(w http.ResponseWriter, status int, errType, message string) { resp := ErrorResponse{ Error: ErrorModel{ Message: message, Type: errType, Code: status, }, } writeJSON(w, status, resp) } // nameValidationError reports whether err is an S3 Tables namespace or table // name validation failure. Such names are client input (the S3 Tables charset // is stricter than the Iceberg REST spec), so the catalog answers 400 rather // than treating it as a server fault. func nameValidationError(err error) bool { if err == nil { return false } // Match the validator's own phrasings rather than a bare "namespace name"/ // "table name" so unrelated faults (e.g. "failed to resolve table name") // aren't misreported as client errors. Lowercased for resilience to // capitalization changes. msg := strings.ToLower(err.Error()) for _, marker := range []string{ "invalid namespace name", "namespace name must", "namespace name cannot", "invalid table name", "table name must", "table name cannot", } { if strings.Contains(msg, marker) { return true } } return false } // writeManagerError maps a residual error to a response: name validation // failures and rejected schemas come from the request body, so they are client // errors (400) and must carry the reason -- a variant field without // format-version 3 otherwise reads as a bare 500 with "variant is not supported // until v3" only in the log. Anything else is a server fault (500). func writeManagerError(w http.ResponseWriter, err error) { switch { case nameValidationError(err), errors.Is(err, iceberg.ErrInvalidSchema), errors.Is(err, iceberg.ErrInvalidPartitionSpec), errors.Is(err, iceberg.ErrInvalidTypeString), errors.Is(err, iceberg.ErrInvalidTransform), errors.Is(err, iceberg.ErrInvalidArgument): writeError(w, http.StatusBadRequest, "BadRequestException", err.Error()) return } // Storage-layer failures are mostly the client's, not the server's. Reporting // a missing namespace or a name conflict as a 500 makes the catalog look // broken and gives the client nothing to act on. var tableErr *s3tables.S3TablesError if errors.As(err, &tableErr) { switch tableErr.Type { case s3tables.ErrCodeNoSuchBucket: // The storage-layer message names the resolved bucket, which for a // client that sent no warehouse at all is the default one it never // asked for, so say how to select a real table bucket. writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("%s: each table bucket is a separate catalog, select one with warehouse=s3:/// or /v1//", tableErr.Message)) case s3tables.ErrCodeNoSuchNamespace: writeError(w, http.StatusNotFound, "NoSuchNamespaceException", tableErr.Message) case s3tables.ErrCodeNoSuchTable: writeError(w, http.StatusNotFound, "NoSuchTableException", tableErr.Message) case s3tables.ErrCodeNoSuchView: writeError(w, http.StatusNotFound, "NoSuchViewException", tableErr.Message) case s3tables.ErrCodeNamespaceAlreadyExists, s3tables.ErrCodeBucketAlreadyExists, s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeViewAlreadyExists: writeError(w, http.StatusConflict, "AlreadyExistsException", tableErr.Message) case s3tables.ErrCodeNamespaceNotEmpty, s3tables.ErrCodeBucketNotEmpty: writeError(w, http.StatusConflict, "AlreadyExistsException", tableErr.Message) case s3tables.ErrCodeConflict: writeError(w, http.StatusConflict, "CommitFailedException", tableErr.Message) case s3tables.ErrCodeAccessDenied: writeError(w, http.StatusForbidden, "ForbiddenException", tableErr.Message) case s3tables.ErrCodeInvalidRequest, s3tables.ErrCodeInvalidIcebergLayout: writeError(w, http.StatusBadRequest, "BadRequestException", tableErr.Message) default: writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error()) } return } writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error()) } // resolveWarehouseBucket maps a client-supplied warehouse value to a table // bucket name. Clients spell the warehouse three ways: the s3:/// // location this catalog advertises, the table bucket ARN that AWS S3 Tables // uses, and the bare bucket name. Accepting only the first sends every other // spelling to the default bucket, where the request fails naming a bucket the // client never asked for. Returns "" when the value names no usable bucket so // the caller keeps its own default. func resolveWarehouseBucket(warehouse string) string { warehouse = strings.TrimSpace(warehouse) var bucket string switch { case warehouse == "": return "" case strings.HasPrefix(warehouse, "s3://"): parsed, _, err := parseS3Location(warehouse) if err != nil { return "" } bucket = parsed case strings.HasPrefix(warehouse, "arn:"): parsed, err := s3tables.ParseBucketNameFromARN(warehouse) if err != nil { return "" } bucket = parsed default: // Bare name, possibly with a sub-path that bucket-scoped routing ignores. bucket, _, _ = strings.Cut(strings.TrimSuffix(warehouse, "/"), "/") } if !s3tables.IsValidBucketName(bucket) { return "" } return bucket } // getBucketFromPrefix extracts table bucket name from prefix parameter. // For now, we use the prefix as the table bucket name. // // The Iceberg REST spec lets clients identify a catalog either by embedding // its prefix in the URL (/v1/{prefix}/...) or by passing ?warehouse= as a // query parameter. Clients that skip the /v1/config handshake (or ignore its // overrides) still routinely send the warehouse parameter on every request, // so honor it as a fallback before the env-var default. func getBucketFromPrefix(r *http.Request) string { vars := mux.Vars(r) if prefix := vars["prefix"]; prefix != "" { return prefix } if bucket := resolveWarehouseBucket(r.URL.Query().Get("warehouse")); bucket != "" { return bucket } if bucket := os.Getenv("S3TABLES_DEFAULT_BUCKET"); bucket != "" { return bucket } // Some writers commit to the unprefixed path even though their reads // honor the /v1/config prefix; the deployment's first table bucket is // where those commits belong. for _, name := range strings.Split(os.Getenv("S3_TABLE_BUCKET"), ",") { if name = strings.TrimSpace(name); name != "" { return name } } // Default bucket if no prefix - use "warehouse" for Iceberg return "warehouse" } // buildTableBucketARN builds an ARN for a table bucket. func buildTableBucketARN(bucketName string) string { arn, _ := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucketName) return arn } const ( defaultListPageSize = 1000 maxListPageSize = 1000 ) func getPaginationQueryParam(r *http.Request, primary, fallback string) string { if v := strings.TrimSpace(r.URL.Query().Get(primary)); v != "" { return v } return strings.TrimSpace(r.URL.Query().Get(fallback)) } func parsePagination(r *http.Request) (pageToken string, pageSize int, err error) { pageToken = getPaginationQueryParam(r, "pageToken", "page-token") pageSize = defaultListPageSize pageSizeValue := getPaginationQueryParam(r, "pageSize", "page-size") if pageSizeValue == "" { return pageToken, pageSize, nil } parsedPageSize, parseErr := strconv.Atoi(pageSizeValue) if parseErr != nil || parsedPageSize <= 0 { return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be a positive integer", pageSizeValue) } if parsedPageSize > maxListPageSize { return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be <= %d", pageSizeValue, maxListPageSize) } return pageToken, parsedPageSize, nil } func normalizeNamespaceProperties(properties map[string]string) map[string]string { if properties == nil { return map[string]string{} } return properties } // namespaceLocationProperty is the well-known Iceberg key used to advertise a // default base path for tables created in a namespace. const namespaceLocationProperty = "location" // defaultNamespaceLocation returns the s3 path under which tables in the // namespace should be stored when the namespace itself does not carry an // explicit "location" property. The flattened namespace path mirrors the on- // disk layout used by handleCreateTable so a deferred client-side commit ends // up at the same place a server-computed one would. func defaultNamespaceLocation(bucket string, namespace []string) string { if bucket == "" || len(namespace) == 0 { return "" } return fmt.Sprintf("s3://%s/%s", bucket, flattenNamespacePath(namespace)) } // withDefaultNamespaceLocation populates the Iceberg "location" property on a // namespace response when the storage layer does not carry one. Trino's REST // catalog falls back to an eager createTable code path when the namespace // has no location, which races our metadata write against Trino's emptiness // check and surfaces as a "Cannot create a table on a non-empty location" // error on the very first CREATE TABLE. Advertising a default location lets // Trino take the deferred-transaction path and pick a unique per-table path. // See issue #9074. func withDefaultNamespaceLocation(properties map[string]string, bucket string, namespace []string) map[string]string { properties = normalizeNamespaceProperties(properties) if _, ok := properties[namespaceLocationProperty]; ok { return properties } if def := defaultNamespaceLocation(bucket, namespace); def != "" { properties[namespaceLocationProperty] = def } return properties }